corosync  2.4.3
totemudpu.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2005 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8 
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <sys/mman.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #include <sys/socket.h>
43 #include <netdb.h>
44 #include <sys/un.h>
45 #include <sys/ioctl.h>
46 #include <sys/param.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 #include <sys/time.h>
57 #include <sys/poll.h>
58 #include <sys/uio.h>
59 #include <limits.h>
60 
61 #include <qb/qbdefs.h>
62 #include <qb/qbloop.h>
63 
64 #include <corosync/sq.h>
65 #include <corosync/list.h>
66 #include <corosync/swab.h>
67 #define LOGSYS_UTILS_ONLY 1
68 #include <corosync/logsys.h>
69 #include "totemudpu.h"
70 
71 #include "util.h"
72 #include "totemcrypto.h"
73 
74 #include <nss.h>
75 #include <pk11pub.h>
76 #include <pkcs11.h>
77 #include <prerror.h>
78 
79 #ifndef MSG_NOSIGNAL
80 #define MSG_NOSIGNAL 0
81 #endif
82 
83 #define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
84 #define NETIF_STATE_REPORT_UP 1
85 #define NETIF_STATE_REPORT_DOWN 2
86 
87 #define BIND_STATE_UNBOUND 0
88 #define BIND_STATE_REGULAR 1
89 #define BIND_STATE_LOOPBACK 2
90 
92  struct list_head list;
94  int fd;
95  int active;
96 };
97 
100 
102 
104 
106 
108 
109  void *context;
110 
111  void (*totemudpu_deliver_fn) (
112  void *context,
113  const void *msg,
114  unsigned int msg_len);
115 
116  void (*totemudpu_iface_change_fn) (
117  void *context,
118  const struct totem_ip_address *iface_address);
119 
120  void (*totemudpu_target_set_completed) (void *context);
121 
122  /*
123  * Function and data used to log messages
124  */
126 
128 
130 
132 
134 
136 
137  void (*totemudpu_log_printf) (
138  int level,
139  int subsys,
140  const char *function,
141  const char *file,
142  int line,
143  const char *format,
144  ...)__attribute__((format(printf, 6, 7)));
145 
146  void *udpu_context;
147 
148  char iov_buffer[FRAME_SIZE_MAX];
149 
150  struct iovec totemudpu_iov_recv;
151 
152  struct list_head member_list;
153 
155 
157 
159 
161 
163 
164  struct timeval stats_tv_start;
165 
166  struct totem_ip_address my_id;
167 
168  int firstrun;
169 
170  qb_loop_timer_handle timer_netif_check_timeout;
171 
172  unsigned int my_memb_entries;
173 
175 
177 
178  struct totem_ip_address token_target;
179 
181 
182  qb_loop_timer_handle timer_merge_detect_timeout;
183 
185 
187 };
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
195 static int totemudpu_build_sockets (
196  struct totemudpu_instance *instance,
197  struct totem_ip_address *bindnet_address,
198  struct totem_ip_address *bound_to);
199 
200 static int totemudpu_create_sending_socket(
201  void *udpu_context,
202  const struct totem_ip_address *member);
203 
205  void *udpu_context);
206 
207 static void totemudpu_start_merge_detect_timeout(
208  void *udpu_context);
209 
210 static void totemudpu_stop_merge_detect_timeout(
211  void *udpu_context);
212 
213 static struct totem_ip_address localhost;
214 
215 static void totemudpu_instance_initialize (struct totemudpu_instance *instance)
216 {
217  memset (instance, 0, sizeof (struct totemudpu_instance));
218 
220 
221  instance->totemudpu_iov_recv.iov_base = instance->iov_buffer;
222 
223  instance->totemudpu_iov_recv.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
224 
225  /*
226  * There is always atleast 1 processor
227  */
228  instance->my_memb_entries = 1;
229 
230  list_init (&instance->member_list);
231 }
232 
233 #define log_printf(level, format, args...) \
234 do { \
235  instance->totemudpu_log_printf ( \
236  level, instance->totemudpu_subsys_id, \
237  __FUNCTION__, __FILE__, __LINE__, \
238  (const char *)format, ##args); \
239 } while (0);
240 #define LOGSYS_PERROR(err_num, level, fmt, args...) \
241 do { \
242  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
243  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
244  instance->totemudpu_log_printf ( \
245  level, instance->totemudpu_subsys_id, \
246  __FUNCTION__, __FILE__, __LINE__, \
247  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
248  } while(0)
249 
251  void *udpu_context,
252  const char *cipher_type,
253  const char *hash_type)
254 {
255 
256  return (0);
257 }
258 
259 
260 static inline void ucast_sendmsg (
261  struct totemudpu_instance *instance,
262  struct totem_ip_address *system_to,
263  const void *msg,
264  unsigned int msg_len)
265 {
266  struct msghdr msg_ucast;
267  int res = 0;
268  size_t buf_out_len;
269  unsigned char buf_out[FRAME_SIZE_MAX];
270  struct sockaddr_storage sockaddr;
271  struct iovec iovec;
272  int addrlen;
273 
274  /*
275  * Encrypt and digest the message
276  */
278  instance->crypto_inst,
279  (const unsigned char *)msg,
280  msg_len,
281  buf_out,
282  &buf_out_len) != 0) {
283  log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
284  return;
285  }
286 
287  iovec.iov_base = (void *)buf_out;
288  iovec.iov_len = buf_out_len;
289 
290  /*
291  * Build unicast message
292  */
294  instance->totem_interface->ip_port, &sockaddr, &addrlen);
295  memset(&msg_ucast, 0, sizeof(msg_ucast));
296  msg_ucast.msg_name = &sockaddr;
297  msg_ucast.msg_namelen = addrlen;
298  msg_ucast.msg_iov = (void *)&iovec;
299  msg_ucast.msg_iovlen = 1;
300 #ifdef HAVE_MSGHDR_CONTROL
301  msg_ucast.msg_control = 0;
302 #endif
303 #ifdef HAVE_MSGHDR_CONTROLLEN
304  msg_ucast.msg_controllen = 0;
305 #endif
306 #ifdef HAVE_MSGHDR_FLAGS
307  msg_ucast.msg_flags = 0;
308 #endif
309 #ifdef HAVE_MSGHDR_ACCRIGHTS
310  msg_ucast.msg_accrights = NULL;
311 #endif
312 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
313  msg_ucast.msg_accrightslen = 0;
314 #endif
315 
316 
317  /*
318  * Transmit unicast message
319  * An error here is recovered by totemsrp
320  */
321  res = sendmsg (instance->token_socket, &msg_ucast, MSG_NOSIGNAL);
322  if (res < 0) {
323  LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
324  "sendmsg(ucast) failed (non-critical)");
325  }
326 }
327 
328 static inline void mcast_sendmsg (
329  struct totemudpu_instance *instance,
330  const void *msg,
331  unsigned int msg_len,
332  int only_active)
333 {
334  struct msghdr msg_mcast;
335  int res = 0;
336  size_t buf_out_len;
337  unsigned char buf_out[FRAME_SIZE_MAX];
338  struct iovec iovec;
339  struct sockaddr_storage sockaddr;
340  int addrlen;
341  struct list_head *list;
342  struct totemudpu_member *member;
343 
344  /*
345  * Encrypt and digest the message
346  */
348  instance->crypto_inst,
349  (const unsigned char *)msg,
350  msg_len,
351  buf_out,
352  &buf_out_len) != 0) {
353  log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
354  return;
355  }
356 
357  iovec.iov_base = (void *)buf_out;
358  iovec.iov_len = buf_out_len;
359 
360  memset(&msg_mcast, 0, sizeof(msg_mcast));
361  /*
362  * Build multicast message
363  */
364  for (list = instance->member_list.next;
365  list != &instance->member_list;
366  list = list->next) {
367 
368  member = list_entry (list,
369  struct totemudpu_member,
370  list);
371 
372  /*
373  * Do not send multicast message if message is not "flush", member
374  * is inactive and timeout for sending merge message didn't expired.
375  */
376  if (only_active && !member->active && !instance->send_merge_detect_message)
377  continue ;
378 
380  instance->totem_interface->ip_port, &sockaddr, &addrlen);
381  msg_mcast.msg_name = &sockaddr;
382  msg_mcast.msg_namelen = addrlen;
383  msg_mcast.msg_iov = (void *)&iovec;
384  msg_mcast.msg_iovlen = 1;
385  #ifdef HAVE_MSGHDR_CONTROL
386  msg_mcast.msg_control = 0;
387  #endif
388  #ifdef HAVE_MSGHDR_CONTROLLEN
389  msg_mcast.msg_controllen = 0;
390  #endif
391  #ifdef HAVE_MSGHDR_FLAGS
392  msg_mcast.msg_flags = 0;
393  #endif
394  #ifdef HAVE_MSGHDR_ACCRIGHTS
395  msg_mcast.msg_accrights = NULL;
396  #endif
397  #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
398  msg_mcast.msg_accrightslen = 0;
399  #endif
400 
401  /*
402  * Transmit multicast message
403  * An error here is recovered by totemsrp
404  */
405  res = sendmsg (member->fd, &msg_mcast, MSG_NOSIGNAL);
406  if (res < 0) {
407  LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
408  "sendmsg(mcast) failed (non-critical)");
409  }
410  }
411 
412  if (!only_active || instance->send_merge_detect_message) {
413  /*
414  * Current message was sent to all nodes
415  */
417  instance->send_merge_detect_message = 0;
418  }
419 }
420 
422  void *udpu_context)
423 {
424  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
425  int res = 0;
426 
427  if (instance->token_socket > 0) {
428  qb_loop_poll_del (instance->totemudpu_poll_handle,
429  instance->token_socket);
430  close (instance->token_socket);
431  }
432 
433  totemudpu_stop_merge_detect_timeout(instance);
434 
435  return (res);
436 }
437 
438 static int net_deliver_fn (
439  int fd,
440  int revents,
441  void *data)
442 {
443  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
444  struct msghdr msg_recv;
445  struct iovec *iovec;
446  struct sockaddr_storage system_from;
447  int bytes_received;
448  int res = 0;
449 
450  iovec = &instance->totemudpu_iov_recv;
451 
452  /*
453  * Receive datagram
454  */
455  msg_recv.msg_name = &system_from;
456  msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
457  msg_recv.msg_iov = iovec;
458  msg_recv.msg_iovlen = 1;
459 #ifdef HAVE_MSGHDR_CONTROL
460  msg_recv.msg_control = 0;
461 #endif
462 #ifdef HAVE_MSGHDR_CONTROLLEN
463  msg_recv.msg_controllen = 0;
464 #endif
465 #ifdef HAVE_MSGHDR_FLAGS
466  msg_recv.msg_flags = 0;
467 #endif
468 #ifdef HAVE_MSGHDR_ACCRIGHTS
469  msg_recv.msg_accrights = NULL;
470 #endif
471 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
472  msg_recv.msg_accrightslen = 0;
473 #endif
474 
475  bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
476  if (bytes_received == -1) {
477  return (0);
478  } else {
479  instance->stats_recv += bytes_received;
480  }
481 
482  /*
483  * Authenticate and if authenticated, decrypt datagram
484  */
485 
486  res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
487  if (res == -1) {
488  log_printf (instance->totemudpu_log_level_security, "Received message has invalid digest... ignoring.");
490  "Invalid packet data");
491  iovec->iov_len = FRAME_SIZE_MAX;
492  return 0;
493  }
494  iovec->iov_len = bytes_received;
495 
496  /*
497  * Handle incoming message
498  */
499  instance->totemudpu_deliver_fn (
500  instance->context,
501  iovec->iov_base,
502  iovec->iov_len);
503 
504  iovec->iov_len = FRAME_SIZE_MAX;
505  return (0);
506 }
507 
508 static int netif_determine (
509  struct totemudpu_instance *instance,
510  struct totem_ip_address *bindnet,
511  struct totem_ip_address *bound_to,
512  int *interface_up,
513  int *interface_num)
514 {
515  int res;
516 
517  res = totemip_iface_check (bindnet, bound_to,
518  interface_up, interface_num,
519  instance->totem_config->clear_node_high_bit);
520 
521 
522  return (res);
523 }
524 
525 
526 /*
527  * If the interface is up, the sockets for totem are built. If the interface is down
528  * this function is requeued in the timer list to retry building the sockets later.
529  */
530 static void timer_function_netif_check_timeout (
531  void *data)
532 {
533  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
534  int interface_up;
535  int interface_num;
536  struct totem_ip_address *bind_address;
537 
538  /*
539  * Build sockets for every interface
540  */
541  netif_determine (instance,
542  &instance->totem_interface->bindnet,
543  &instance->totem_interface->boundto,
544  &interface_up, &interface_num);
545  /*
546  * If the network interface isn't back up and we are already
547  * in loopback mode, add timer to check again and return
548  */
549  if ((instance->netif_bind_state == BIND_STATE_LOOPBACK &&
550  interface_up == 0) ||
551 
552  (instance->my_memb_entries == 1 &&
553  instance->netif_bind_state == BIND_STATE_REGULAR &&
554  interface_up == 1)) {
555 
556  qb_loop_timer_add (instance->totemudpu_poll_handle,
557  QB_LOOP_MED,
558  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
559  (void *)instance,
560  timer_function_netif_check_timeout,
561  &instance->timer_netif_check_timeout);
562 
563  /*
564  * Add a timer to check for a downed regular interface
565  */
566  return;
567  }
568 
569  if (instance->token_socket > 0) {
570  qb_loop_poll_del (instance->totemudpu_poll_handle,
571  instance->token_socket);
572  close (instance->token_socket);
573  }
574 
575  if (interface_up == 0) {
576  /*
577  * Interface is not up
578  */
580  bind_address = &localhost;
581 
582  /*
583  * Add a timer to retry building interfaces and request memb_gather_enter
584  */
585  qb_loop_timer_add (instance->totemudpu_poll_handle,
586  QB_LOOP_MED,
587  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
588  (void *)instance,
589  timer_function_netif_check_timeout,
590  &instance->timer_netif_check_timeout);
591  } else {
592  /*
593  * Interface is up
594  */
596  bind_address = &instance->totem_interface->bindnet;
597  }
598  /*
599  * Create and bind the multicast and unicast sockets
600  */
601  totemudpu_build_sockets (instance,
602  bind_address,
603  &instance->totem_interface->boundto);
604 
605  qb_loop_poll_add (instance->totemudpu_poll_handle,
606  QB_LOOP_MED,
607  instance->token_socket,
608  POLLIN, instance, net_deliver_fn);
609 
610  totemip_copy (&instance->my_id, &instance->totem_interface->boundto);
611 
612  /*
613  * This reports changes in the interface to the user and totemsrp
614  */
615  if (instance->netif_bind_state == BIND_STATE_REGULAR) {
616  if (instance->netif_state_report & NETIF_STATE_REPORT_UP) {
618  "The network interface [%s] is now up.",
619  totemip_print (&instance->totem_interface->boundto));
621  instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
622  }
623  /*
624  * Add a timer to check for interface going down in single membership
625  */
626  if (instance->my_memb_entries == 1) {
627  qb_loop_timer_add (instance->totemudpu_poll_handle,
628  QB_LOOP_MED,
629  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
630  (void *)instance,
631  timer_function_netif_check_timeout,
632  &instance->timer_netif_check_timeout);
633  }
634 
635  } else {
638  "The network interface is down.");
639  instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
640  }
642 
643  }
644 }
645 
646 /* Set the socket priority to INTERACTIVE to ensure
647  that our messages don't get queued behind anything else */
648 static void totemudpu_traffic_control_set(struct totemudpu_instance *instance, int sock)
649 {
650 #ifdef SO_PRIORITY
651  int prio = 6; /* TC_PRIO_INTERACTIVE */
652 
653  if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(int))) {
654  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
655  "Could not set traffic priority");
656  }
657 #endif
658 }
659 
660 static int totemudpu_build_sockets_ip (
661  struct totemudpu_instance *instance,
662  struct totem_ip_address *bindnet_address,
663  struct totem_ip_address *bound_to,
664  int interface_num)
665 {
666  struct sockaddr_storage sockaddr;
667  int addrlen;
668  int res;
669  unsigned int recvbuf_size;
670  unsigned int optlen = sizeof (recvbuf_size);
671  unsigned int retries = 0;
672 
673  /*
674  * Setup unicast socket
675  */
676  instance->token_socket = socket (bindnet_address->family, SOCK_DGRAM, 0);
677  if (instance->token_socket == -1) {
678  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
679  "socket() failed");
680  return (-1);
681  }
682 
683  totemip_nosigpipe (instance->token_socket);
684  res = fcntl (instance->token_socket, F_SETFL, O_NONBLOCK);
685  if (res == -1) {
686  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
687  "Could not set non-blocking operation on token socket");
688  return (-1);
689  }
690 
691  /*
692  * Bind to unicast socket used for token send/receives
693  * This has the side effect of binding to the correct interface
694  */
695  totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &sockaddr, &addrlen);
696  while (1) {
697  res = bind (instance->token_socket, (struct sockaddr *)&sockaddr, addrlen);
698  if (res == 0) {
699  break;
700  }
701  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
702  "bind token socket failed");
703  if (++retries > BIND_MAX_RETRIES) {
704  break;
705  }
706 
707  /*
708  * Wait for a while
709  */
710  (void)poll(NULL, 0, BIND_RETRIES_INTERVAL * retries);
711  }
712 
713  if (res == -1) {
714  return (-1);
715  }
716 
717  /*
718  * the token_socket can receive many messages. Allow a large number
719  * of receive messages on this socket
720  */
721  recvbuf_size = MCAST_SOCKET_BUFFER_SIZE;
722  res = setsockopt (instance->token_socket, SOL_SOCKET, SO_RCVBUF,
723  &recvbuf_size, optlen);
724  if (res == -1) {
725  LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
726  "Could not set recvbuf size");
727  }
728 
729  return 0;
730 }
731 
732 static int totemudpu_build_sockets (
733  struct totemudpu_instance *instance,
734  struct totem_ip_address *bindnet_address,
735  struct totem_ip_address *bound_to)
736 {
737  int interface_num;
738  int interface_up;
739  int res;
740 
741  /*
742  * Determine the ip address bound to and the interface name
743  */
744  res = netif_determine (instance,
745  bindnet_address,
746  bound_to,
747  &interface_up,
748  &interface_num);
749 
750  if (res == -1) {
751  return (-1);
752  }
753 
754  totemip_copy(&instance->my_id, bound_to);
755 
756  res = totemudpu_build_sockets_ip (instance,
757  bindnet_address, bound_to, interface_num);
758 
759  if (res == -1) {
760  /* if we get here, corosync won't work anyway, so better leaving than faking to work */
761  LOGSYS_PERROR (errno, instance->totemudpu_log_level_error,
762  "Unable to create sockets, exiting");
763  exit(EXIT_FAILURE);
764  }
765 
766  /* We only send out of the token socket */
767  totemudpu_traffic_control_set(instance, instance->token_socket);
768 
769  /*
770  * Rebind all members to new ips
771  */
773 
774  return res;
775 }
776 
777 /*
778  * Totem Network interface - also does encryption/decryption
779  * depends on poll abstraction, POSIX, IPV4
780  */
781 
782 /*
783  * Create an instance
784  */
786  qb_loop_t *poll_handle,
787  void **udpu_context,
788  struct totem_config *totem_config,
789  totemsrp_stats_t *stats,
790  int interface_no,
791  void *context,
792 
793  void (*deliver_fn) (
794  void *context,
795  const void *msg,
796  unsigned int msg_len),
797 
798  void (*iface_change_fn) (
799  void *context,
800  const struct totem_ip_address *iface_address),
801 
802  void (*target_set_completed) (
803  void *context))
804 {
805  struct totemudpu_instance *instance;
806 
807  instance = malloc (sizeof (struct totemudpu_instance));
808  if (instance == NULL) {
809  return (-1);
810  }
811 
812  totemudpu_instance_initialize (instance);
813 
814  instance->totem_config = totem_config;
815  instance->stats = stats;
816 
817  /*
818  * Configure logging
819  */
820  instance->totemudpu_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
827 
828  /*
829  * Initialize random number generator for later use to generate salt
830  */
831  instance->crypto_inst = crypto_init (totem_config->private_key,
832  totem_config->private_key_len,
833  totem_config->crypto_cipher_type,
834  totem_config->crypto_hash_type,
835  instance->totemudpu_log_printf,
837  instance->totemudpu_log_level_notice,
838  instance->totemudpu_log_level_error,
839  instance->totemudpu_subsys_id);
840  if (instance->crypto_inst == NULL) {
841  free(instance);
842  return (-1);
843  }
844  /*
845  * Initialize local variables for totemudpu
846  */
847  instance->totem_interface = &totem_config->interfaces[interface_no];
848  memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
849 
850  instance->totemudpu_poll_handle = poll_handle;
851 
852  instance->totem_interface->bindnet.nodeid = instance->totem_config->node_id;
853 
854  instance->context = context;
855  instance->totemudpu_deliver_fn = deliver_fn;
856 
857  instance->totemudpu_iface_change_fn = iface_change_fn;
858 
859  instance->totemudpu_target_set_completed = target_set_completed;
860 
861  totemip_localhost (AF_INET, &localhost);
862  localhost.nodeid = instance->totem_config->node_id;
863 
864  /*
865  * RRP layer isn't ready to receive message because it hasn't
866  * initialized yet. Add short timer to check the interfaces.
867  */
868  qb_loop_timer_add (instance->totemudpu_poll_handle,
869  QB_LOOP_MED,
870  100*QB_TIME_NS_IN_MSEC,
871  (void *)instance,
872  timer_function_netif_check_timeout,
873  &instance->timer_netif_check_timeout);
874 
875  totemudpu_start_merge_detect_timeout(instance);
876 
877  *udpu_context = instance;
878  return (0);
879 }
880 
882 {
883  return malloc (FRAME_SIZE_MAX);
884 }
885 
886 void totemudpu_buffer_release (void *ptr)
887 {
888  return free (ptr);
889 }
890 
892  void *udpu_context,
893  int processor_count)
894 {
895  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
896  int res = 0;
897 
898  instance->my_memb_entries = processor_count;
899  qb_loop_timer_del (instance->totemudpu_poll_handle,
900  instance->timer_netif_check_timeout);
901  if (processor_count == 1) {
902  qb_loop_timer_add (instance->totemudpu_poll_handle,
903  QB_LOOP_MED,
904  instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
905  (void *)instance,
906  timer_function_netif_check_timeout,
907  &instance->timer_netif_check_timeout);
908  }
909 
910  return (res);
911 }
912 
913 int totemudpu_recv_flush (void *udpu_context)
914 {
915  int res = 0;
916 
917  return (res);
918 }
919 
920 int totemudpu_send_flush (void *udpu_context)
921 {
922  int res = 0;
923 
924  return (res);
925 }
926 
928  void *udpu_context,
929  const void *msg,
930  unsigned int msg_len)
931 {
932  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
933  int res = 0;
934 
935  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
936 
937  return (res);
938 }
940  void *udpu_context,
941  const void *msg,
942  unsigned int msg_len)
943 {
944  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
945  int res = 0;
946 
947  mcast_sendmsg (instance, msg, msg_len, 0);
948 
949  return (res);
950 }
951 
953  void *udpu_context,
954  const void *msg,
955  unsigned int msg_len)
956 {
957  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
958  int res = 0;
959 
960  mcast_sendmsg (instance, msg, msg_len, 1);
961 
962  return (res);
963 }
964 
965 extern int totemudpu_iface_check (void *udpu_context)
966 {
967  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
968  int res = 0;
969 
970  timer_function_netif_check_timeout (instance);
971 
972  return (res);
973 }
974 
975 extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *totem_config)
976 {
977 
978  assert(totem_config->interface_count > 0);
979 
980  totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
981  totem_config->crypto_hash_type) +
983 }
984 
985 const char *totemudpu_iface_print (void *udpu_context) {
986  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
987  const char *ret_char;
988 
989  ret_char = totemip_print (&instance->my_id);
990 
991  return (ret_char);
992 }
993 
995  void *udpu_context,
996  struct totem_ip_address *addr)
997 {
998  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
999  int res = 0;
1000 
1001  memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
1002 
1003  return (res);
1004 }
1005 
1007  void *udpu_context,
1008  const struct totem_ip_address *token_target)
1009 {
1010  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1011  int res = 0;
1012 
1013  memcpy (&instance->token_target, token_target,
1014  sizeof (struct totem_ip_address));
1015 
1016  instance->totemudpu_target_set_completed (instance->context);
1017 
1018  return (res);
1019 }
1020 
1022  void *udpu_context)
1023 {
1024  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1025  unsigned int res;
1026  struct sockaddr_storage system_from;
1027  struct msghdr msg_recv;
1028  struct pollfd ufd;
1029  int nfds;
1030  int msg_processed = 0;
1031 
1032  /*
1033  * Receive datagram
1034  */
1035  msg_recv.msg_name = &system_from;
1036  msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
1037  msg_recv.msg_iov = &instance->totemudpu_iov_recv;
1038  msg_recv.msg_iovlen = 1;
1039 #ifdef HAVE_MSGHDR_CONTROL
1040  msg_recv.msg_control = 0;
1041 #endif
1042 #ifdef HAVE_MSGHDR_CONTROLLEN
1043  msg_recv.msg_controllen = 0;
1044 #endif
1045 #ifdef HAVE_MSGHDR_FLAGS
1046  msg_recv.msg_flags = 0;
1047 #endif
1048 #ifdef HAVE_MSGHDR_ACCRIGHTS
1049  msg_recv.msg_accrights = NULL;
1050 #endif
1051 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1052  msg_recv.msg_accrightslen = 0;
1053 #endif
1054 
1055  do {
1056  ufd.fd = instance->token_socket;
1057  ufd.events = POLLIN;
1058  nfds = poll (&ufd, 1, 0);
1059  if (nfds == 1 && ufd.revents & POLLIN) {
1060  res = recvmsg (instance->token_socket, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
1061  if (res != -1) {
1062  msg_processed = 1;
1063  } else {
1064  msg_processed = -1;
1065  }
1066  }
1067  } while (nfds == 1);
1068 
1069  return (msg_processed);
1070 }
1071 
1072 static int totemudpu_create_sending_socket(
1073  void *udpu_context,
1074  const struct totem_ip_address *member)
1075 {
1076  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1077  int fd;
1078  int res;
1079  unsigned int sendbuf_size;
1080  unsigned int optlen = sizeof (sendbuf_size);
1081  struct sockaddr_storage sockaddr;
1082  int addrlen;
1083 
1084  fd = socket (member->family, SOCK_DGRAM, 0);
1085  if (fd == -1) {
1086  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1087  "Could not create socket for new member");
1088  return (-1);
1089  }
1090  totemip_nosigpipe (fd);
1091  res = fcntl (fd, F_SETFL, O_NONBLOCK);
1092  if (res == -1) {
1093  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1094  "Could not set non-blocking operation on token socket");
1095  goto error_close_fd;
1096  }
1097 
1098  /*
1099  * These sockets are used to send multicast messages, so their buffers
1100  * should be large
1101  */
1102  sendbuf_size = MCAST_SOCKET_BUFFER_SIZE;
1103  res = setsockopt (fd, SOL_SOCKET, SO_SNDBUF,
1104  &sendbuf_size, optlen);
1105  if (res == -1) {
1106  LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
1107  "Could not set sendbuf size");
1108  /*
1109  * Fail in setting sendbuf size is not fatal -> don't exit
1110  */
1111  }
1112 
1113  /*
1114  * Bind to sending interface
1115  */
1116  totemip_totemip_to_sockaddr_convert(&instance->my_id, 0, &sockaddr, &addrlen);
1117  res = bind (fd, (struct sockaddr *)&sockaddr, addrlen);
1118  if (res == -1) {
1119  LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
1120  "bind token socket failed");
1121  goto error_close_fd;
1122  }
1123 
1124  return (fd);
1125 
1126 error_close_fd:
1127  close(fd);
1128  return (-1);
1129 }
1130 
1132  void *udpu_context,
1133  const struct totem_ip_address *member)
1134 {
1135  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1136 
1137  struct totemudpu_member *new_member;
1138 
1139  new_member = malloc (sizeof (struct totemudpu_member));
1140  if (new_member == NULL) {
1141  return (-1);
1142  }
1143 
1144  memset(new_member, 0, sizeof(*new_member));
1145 
1146  log_printf (LOGSYS_LEVEL_NOTICE, "adding new UDPU member {%s}",
1147  totemip_print(member));
1148  list_init (&new_member->list);
1149  list_add_tail (&new_member->list, &instance->member_list);
1150  memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
1151  new_member->fd = totemudpu_create_sending_socket(udpu_context, member);
1152  new_member->active = 0;
1153 
1154  return (0);
1155 }
1156 
1158  void *udpu_context,
1159  const struct totem_ip_address *token_target)
1160 {
1161  int found = 0;
1162  struct list_head *list;
1163  struct totemudpu_member *member;
1164 
1165  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1166 
1167  /*
1168  * Find the member to remove and close its socket
1169  */
1170  for (list = instance->member_list.next;
1171  list != &instance->member_list;
1172  list = list->next) {
1173 
1174  member = list_entry (list,
1175  struct totemudpu_member,
1176  list);
1177 
1178  if (totemip_compare (token_target, &member->member)==0) {
1180  "removing UDPU member {%s}",
1181  totemip_print(&member->member));
1182 
1183  if (member->fd > 0) {
1185  "Closing socket to: {%s}",
1186  totemip_print(&member->member));
1187  qb_loop_poll_del (instance->totemudpu_poll_handle,
1188  member->fd);
1189  close (member->fd);
1190  }
1191  found = 1;
1192  break;
1193  }
1194  }
1195 
1196  /*
1197  * Delete the member from the list
1198  */
1199  if (found) {
1200  list_del (list);
1201  }
1202 
1203  instance = NULL;
1204  return (0);
1205 }
1206 
1208  void *udpu_context)
1209 {
1210  struct list_head *list;
1211  struct totemudpu_member *member;
1212 
1213  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1214 
1215  for (list = instance->member_list.next;
1216  list != &instance->member_list;
1217  list = list->next) {
1218 
1219  member = list_entry (list,
1220  struct totemudpu_member,
1221  list);
1222 
1223  if (member->fd > 0) {
1224  close (member->fd);
1225  }
1226 
1227  member->fd = totemudpu_create_sending_socket(udpu_context, &member->member);
1228  }
1229 
1230  return (0);
1231 }
1232 
1234  void *udpu_context,
1235  const struct totem_ip_address *member_ip,
1236  int active)
1237 {
1238  struct list_head *list;
1239  struct totemudpu_member *member;
1240  int addr_found = 0;
1241 
1242  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1243 
1244  /*
1245  * Find the member to set active flag
1246  */
1247  for (list = instance->member_list.next; list != &instance->member_list; list = list->next) {
1248  member = list_entry (list, struct totemudpu_member, list);
1249 
1250  if (totemip_compare (member_ip, &member->member) == 0) {
1252  "Marking UDPU member %s %s",
1253  totemip_print(&member->member),
1254  (active ? "active" : "inactive"));
1255 
1256  member->active = active;
1257  addr_found = 1;
1258 
1259  break;
1260  }
1261  }
1262 
1263  if (!addr_found) {
1265  "Can't find UDPU member %s (should be marked as %s)",
1266  totemip_print(member_ip),
1267  (active ? "active" : "inactive"));
1268  }
1269 
1270  return (0);
1271 }
1272 
1273 static void timer_function_merge_detect_timeout (
1274  void *data)
1275 {
1276  struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
1277 
1278  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1279  instance->send_merge_detect_message = 1;
1280  }
1281 
1283 
1284  totemudpu_start_merge_detect_timeout(instance);
1285 }
1286 
1287 static void totemudpu_start_merge_detect_timeout(
1288  void *udpu_context)
1289 {
1290  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1291 
1292  qb_loop_timer_add(instance->totemudpu_poll_handle,
1293  QB_LOOP_MED,
1294  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1295  (void *)instance,
1296  timer_function_merge_detect_timeout,
1297  &instance->timer_merge_detect_timeout);
1298 
1299 }
1300 
1301 static void totemudpu_stop_merge_detect_timeout(
1302  void *udpu_context)
1303 {
1304  struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
1305 
1306  qb_loop_timer_del(instance->totemudpu_poll_handle,
1307  instance->timer_merge_detect_timeout);
1308 }
unsigned int clear_node_high_bit
Definition: totem.h:120
unsigned short family
Definition: coroapi.h:113
#define NETIF_STATE_REPORT_UP
Definition: totemudpu.c:84
struct totem_config * totem_config
Definition: totemudpu.c:174
struct totem_ip_address member
Definition: totemudpu.c:93
int totemip_localhost(int family, struct totem_ip_address *localhost)
Definition: totemip.c:182
unsigned int my_memb_entries
Definition: totemudpu.c:172
#define BIND_MAX_RETRIES
Definition: totem.h:55
void(* totemudpu_iface_change_fn)(void *context, const struct totem_ip_address *iface_address)
Definition: totemudpu.c:116
struct totem_interface * interfaces
Definition: totem.h:117
unsigned int interface_count
Definition: totem.h:118
struct list_head * next
Definition: list.h:47
size_t crypto_sec_header_size(const char *crypto_cipher_type, const char *crypto_hash_type)
Definition: totemcrypto.c:771
The totem_ip_address struct.
Definition: coroapi.h:111
void(*) void udpu_context)
Definition: totemudpu.c:144
int totemudpu_token_target_set(void *udpu_context, const struct totem_ip_address *token_target)
Definition: totemudpu.c:1006
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:214
struct totem_ip_address my_id
Definition: totemudpu.c:166
struct totemudpu_instance * instance
Definition: totemudpu.c:192
#define NETIF_STATE_REPORT_DOWN
Definition: totemudpu.c:85
int totemip_compare(const void *a, const void *b)
Definition: totemip.c:130
#define log_printf(level, format, args...)
Definition: totemudpu.c:233
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN]
Definition: totem.h:125
int totemudpu_processor_count_set(void *udpu_context, int processor_count)
Definition: totemudpu.c:891
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
int totemudpu_log_level_security
Definition: totemudpu.c:125
struct crypto_instance * crypto_init(const unsigned char *private_key, unsigned int private_key_len, const char *crypto_cipher_type, const char *crypto_hash_type, void(*log_printf_func)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf, 6, 7))), int log_level_security, int log_level_notice, int log_level_error, int log_subsys_id)
Definition: totemcrypto.c:895
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:95
unsigned int downcheck_timeout
Definition: totem.h:148
unsigned int private_key_len
Definition: totem.h:127
char iov_buffer[FRAME_SIZE_MAX]
Definition: totemudpu.c:148
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemudpu.c:182
Definition: list.h:46
int send_merge_detect_message
Definition: totemudpu.c:184
#define totemip_nosigpipe(s)
Definition: totemip.h:56
int totemudpu_log_level_warning
Definition: totemudpu.c:129
int totemudpu_log_level_debug
Definition: totemudpu.c:133
const char * totemudpu_iface_print(void *udpu_context)
Definition: totemudpu.c:985
struct iovec totemudpu_iov_recv
Definition: totemudpu.c:150
unsigned int node_id
Definition: totem.h:119
#define BIND_STATE_REGULAR
Definition: totemudpu.c:88
int totemip_iface_check(struct totem_ip_address *bindnet, struct totem_ip_address *boundto, int *interface_up, int *interface_num, int mask_high_bit)
Definition: totemip.c:405
int crypto_encrypt_and_sign(struct crypto_instance *instance, const unsigned char *buf_in, const size_t buf_in_len, unsigned char *buf_out, size_t *buf_out_len)
Definition: totemcrypto.c:820
void totemudpu_buffer_release(void *ptr)
Definition: totemudpu.c:886
void * totemudpu_buffer_alloc(void)
Definition: totemudpu.c:881
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemudpu.c:186
qb_loop_t * totemudpu_poll_handle
Definition: totemudpu.c:101
int totemudpu_mcast_noflush_send(void *udpu_context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:952
void(* totemudpu_deliver_fn)(void *context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:111
unsigned int nodeid
Definition: coroapi.h:112
char * crypto_hash_type
Definition: totem.h:186
totemsrp_stats_t * stats
Definition: totemudpu.c:176
Linked list API.
#define BIND_RETRIES_INTERVAL
Definition: totem.h:56
struct totem_ip_address token_target
Definition: totemudpu.c:178
int totemudpu_crypto_set(void *udpu_context, const char *cipher_type, const char *hash_type)
Definition: totemudpu.c:250
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
struct totem_interface * totem_interface
Definition: totemudpu.c:103
int totemudpu_token_send(void *udpu_context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:927
struct totem_ip_address boundto
Definition: totem.h:69
typedef __attribute__
size_t totemip_udpip_header_size(int family)
Definition: totemip.c:496
struct list_head member_list
Definition: totemudpu.c:152
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:78
uint16_t ip_port
Definition: totem.h:71
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemudpu.c:170
void(* totemudpu_target_set_completed)(void *context)
Definition: totemudpu.c:120
#define BIND_STATE_LOOPBACK
Definition: totemudpu.c:89
unsigned int net_mtu
Definition: totem.h:168
#define MCAST_SOCKET_BUFFER_SIZE
Definition: totemudpu.c:83
int crypto_authenticate_and_decrypt(struct crypto_instance *instance, unsigned char *buf, int *buf_len)
Definition: totemcrypto.c:842
int totemudpu_member_remove(void *udpu_context, const struct totem_ip_address *token_target)
Definition: totemudpu.c:1157
int totemudpu_member_set_active(void *udpu_context, const struct totem_ip_address *member_ip, int active)
Definition: totemudpu.c:1233
#define FRAME_SIZE_MAX
Definition: totem.h:50
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:69
#define list_entry(ptr, type, member)
Definition: list.h:84
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:222
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:166
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
int totemudpu_recv_mcast_empty(void *udpu_context)
Definition: totemudpu.c:1021
int totemudpu_initialize(qb_loop_t *poll_handle, void **udpu_context, struct totem_config *totem_config, totemsrp_stats_t *stats, int interface_no, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address), void(*target_set_completed)(void *context))
Create an instance.
Definition: totemudpu.c:785
struct srp_addr system_from
Definition: totemsrp.c:61
char * crypto_cipher_type
Definition: totem.h:184
int totemudpu_log_level_error
Definition: totemudpu.c:127
unsigned int merge_timeout
Definition: totem.h:146
void totemudpu_net_mtu_adjust(void *udpu_context, struct totem_config *totem_config)
Definition: totemudpu.c:975
struct totem_ip_address bindnet
Definition: totem.h:68
#define MSG_NOSIGNAL
Definition: totemudpu.c:80
int totemudpu_mcast_flush_send(void *udpu_context, const void *msg, unsigned int msg_len)
Definition: totemudpu.c:939
int totemudpu_send_flush(void *udpu_context)
Definition: totemudpu.c:920
int totemudpu_finalize(void *udpu_context)
Definition: totemudpu.c:421
struct crypto_instance * crypto_inst
Definition: totemudpu.c:99
int totemudpu_recv_flush(void *udpu_context)
Definition: totemudpu.c:913
int totemudpu_member_add(void *udpu_context, const struct totem_ip_address *member)
Definition: totemudpu.c:1131
struct list_head list
Definition: totemudpu.c:92
#define LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemudpu.c:240
int totemudpu_iface_get(void *udpu_context, struct totem_ip_address *addr)
Definition: totemudpu.c:994
int totemudpu_member_list_rebind_ip(void *udpu_context)
Definition: totemudpu.c:1207
void(* totemudpu_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemudpu.c:137
int totemudpu_iface_check(void *udpu_context)
Definition: totemudpu.c:965
int totemudpu_log_level_notice
Definition: totemudpu.c:131