One Level Up Top Level

src/event/quic/ngx_event_quic_bpf.c - nginx source code

Global variables defined

Data types defined

Functions defined

Macros defined

Source code


  2. /*
  3. * Copyright (C) Nginx, Inc.
  4. */


  7. #include <ngx_config.h>
  8. #include <ngx_core.h>


  11. ‌#define NGX_QUIC_BPF_VARNAME  "NGINX_BPF_MAPS"
  12. ‌#define NGX_QUIC_BPF_VARSEP    ';'
  13. ‌#define NGX_QUIC_BPF_ADDRSEP   '#'


  16. ‌#define ngx_quic_bpf_get_conf(cycle)                                          \
  17.     (ngx_quic_bpf_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_quic_bpf_module)

  19. ‌#define ngx_quic_bpf_get_old_conf(cycle)                                      \
  20.     cycle->old_cycle->conf_ctx ? ngx_quic_bpf_get_conf(cycle->old_cycle)      \
  21.                                : NULL

  23. ‌#define ngx_core_get_conf(cycle)                                              \
  24.     (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module)


  27. ‌typedef struct {
  28.     ngx_queue_t           queue;
  29.     int                   map_fd;

  31.     struct sockaddr      *sockaddr;
  32.     socklen_t             socklen;
  33.     ngx_uint_t            unused;     /* unsigned  unused:1; */
  34. ‌} ngx_quic_sock_group_t;


  37. ‌typedef struct {
  38.     ngx_flag_t            enabled;
  39.     ngx_uint_t            map_size;
  40.     ngx_queue_t           groups;     /* of ngx_quic_sock_group_t */
  41. ‌} ngx_quic_bpf_conf_t;


  44. static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle);
  45. static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle);

  47. static void ngx_quic_bpf_cleanup(void *data);
  48. static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd,
  49.     const char *name);

  51. static ngx_quic_sock_group_t *ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf,
  52.     ngx_listening_t *ls);
  53. static ngx_quic_sock_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle,
  54.     struct sockaddr *sa, socklen_t socklen);
  55. static ngx_quic_sock_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle,
  56.     ngx_listening_t *ls);
  57. static ngx_quic_sock_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle,
  58.     ngx_listening_t *ls);
  59. static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle,
  60.     ngx_listening_t *ls);
  61. static uint64_t ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log);

  63. static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle);
  64. static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle);

  66. extern ngx_bpf_program_t  ngx_quic_reuseport_helper;


  69. ‌static ngx_command_t  ngx_quic_bpf_commands[] = {

  71.     { ngx_string("quic_bpf"),
  72.       NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_FLAG,
  73.       ngx_conf_set_flag_slot,
  74.       0,
  75.       offsetof(ngx_quic_bpf_conf_t, enabled),
  76.       NULL },

  78.       ngx_null_command
  79. };


  82. ‌static ngx_core_module_t  ngx_quic_bpf_module_ctx = {
  83.     ngx_string("quic_bpf"),
  84.     ngx_quic_bpf_create_conf,
  85.     NULL
  86. };


  89. ‌ngx_module_t  ngx_quic_bpf_module = {
  90.     NGX_MODULE_V1,
  91.     &ngx_quic_bpf_module_ctx,              /* module context */
  92.     ngx_quic_bpf_commands,                 /* module directives */
  93.     NGX_CORE_MODULE,                       /* module type */
  94.     NULL,                                  /* init master */
  95.     ngx_quic_bpf_module_init,              /* init module */
  96.     NULL,                                  /* init process */
  97.     NULL,                                  /* init thread */
  98.     NULL,                                  /* exit thread */
  99.     NULL,                                  /* exit process */
  100.     NULL,                                  /* exit master */
  101.     NGX_MODULE_V1_PADDING
  102. };


  105. static void *
  106. ‌ngx_quic_bpf_create_conf(ngx_cycle_t *cycle)
  107. {
  108.     ngx_quic_bpf_conf_t  *bcf;

  110.     bcf = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_conf_t));
  111.     if (bcf == NULL) {
  112.         return NULL;
  113.     }

  115.     bcf->enabled = NGX_CONF_UNSET;
  116.     bcf->map_size = NGX_CONF_UNSET_UINT;

  118.     ngx_queue_init(&bcf->groups);

  120.     return bcf;
  121. }


  124. static ngx_int_t
  125. ‌ngx_quic_bpf_module_init(ngx_cycle_t *cycle)
  126. {
  127.     ngx_uint_t            i;
  128.     ngx_listening_t      *ls;
  129.     ngx_core_conf_t      *ccf;
  130.     ngx_pool_cleanup_t   *cln;
  131.     ngx_quic_bpf_conf_t  *bcf;

  133.     if (ngx_test_config) {
  134.         /*
  135.          * during config test, SO_REUSEPORT socket option is
  136.          * not set, thus making further processing meaningless
  137.          */
  138.         return NGX_OK;
  139.     }

  141.     ccf = ngx_core_get_conf(cycle);
  142.     bcf = ngx_quic_bpf_get_conf(cycle);

  144.     ngx_conf_init_value(bcf->enabled, 0);

  146.     bcf->map_size = ccf->worker_processes * 4;

  148.     cln = ngx_pool_cleanup_add(cycle->pool, 0);
  149.     if (cln == NULL) {
  150.         goto failed;
  151.     }

  153.     cln->data = bcf;
  154.     cln->handler = ngx_quic_bpf_cleanup;

  156.     if (ngx_inherited && ngx_is_init_cycle(cycle->old_cycle)) {
  157.         if (ngx_quic_bpf_import_maps(cycle) != NGX_OK) {
  158.             goto failed;
  159.         }
  160.     }

  162.     ls = cycle->listening.elts;

  164.     for (i = 0; i < cycle->listening.nelts; i++) {
  165.         if (ls[i].quic && ls[i].reuseport) {
  166.             if (ngx_quic_bpf_group_add_socket(cycle, &ls[i]) != NGX_OK) {
  167.                 goto failed;
  168.             }
  169.         }
  170.     }

  172.     if (ngx_quic_bpf_export_maps(cycle) != NGX_OK) {
  173.         goto failed;
  174.     }

  176.     return NGX_OK;

  178. failed:

  180.     if (ngx_is_init_cycle(cycle->old_cycle)) {
  181.         ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
  182.                       "ngx_quic_bpf_module failed to initialize, check limits");

  184.         /* refuse to start */
  185.         return NGX_ERROR;
  186.     }

  188.     /*
  189.      * returning error now will lead to master process exiting immediately
  190.      * leaving worker processes orphaned, what is really unexpected.
  191.      * Instead, just issue a not about failed initialization and try
  192.      * to cleanup a bit. Still program can be already loaded to kernel
  193.      * for some reuseport groups, and there is no way to revert, so
  194.      * behaviour may be inconsistent.
  195.      */

  197.     ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
  198.                   "ngx_quic_bpf_module failed to initialize properly, ignored."
  199.                   "please check limits and note that nginx state now "
  200.                   "can be inconsistent and restart may be required");

  202.     return NGX_OK;
  203. }


  206. static void
  207. ‌ngx_quic_bpf_cleanup(void *data)
  208. {
  209.     ngx_quic_bpf_conf_t  *bcf = (ngx_quic_bpf_conf_t *) data;

  211.     ngx_queue_t            *q;
  212.     ngx_quic_sock_group_t  *grp;

  214.     for (q = ngx_queue_head(&bcf->groups);
  215.          q != ngx_queue_sentinel(&bcf->groups);
  216.          q = ngx_queue_next(q))
  217.     {
  218.         grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);

  220.         ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map");
  221.     }
  222. }


  225. static ngx_inline void
  226. ‌ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name)
  227. {
  228.     if (close(fd) != -1) {
  229.         return;
  230.     }

  232.     ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,
  233.                   "quic bpf close %s fd:%d failed", name, fd);
  234. }


  237. static ngx_quic_sock_group_t *
  238. ‌ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls)
  239. {
  240.     ngx_queue_t            *q;
  241.     ngx_quic_sock_group_t  *grp;

  243.     for (q = ngx_queue_head(&bcf->groups);
  244.          q != ngx_queue_sentinel(&bcf->groups);
  245.          q = ngx_queue_next(q))
  246.     {
  247.         grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);

  249.         if (ngx_cmp_sockaddr(ls->sockaddr, ls->socklen,
  250.                              grp->sockaddr, grp->socklen, 1)
  251.             == NGX_OK)
  252.         {
  253.             return grp;
  254.         }
  255.     }

  257.     return NULL;
  258. }


  261. static ngx_quic_sock_group_t *
  262. ‌ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa,
  263.     socklen_t socklen)
  264. {
  265.     ngx_quic_bpf_conf_t    *bcf;
  266.     ngx_quic_sock_group_t  *grp;

  268.     bcf = ngx_quic_bpf_get_conf(cycle);

  270.     grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_sock_group_t));
  271.     if (grp == NULL) {
  272.         return NULL;
  273.     }

  275.     grp->socklen = socklen;
  276.     grp->sockaddr = ngx_palloc(cycle->pool, socklen);
  277.     if (grp->sockaddr == NULL) {
  278.         return NULL;
  279.     }
  280.     ngx_memcpy(grp->sockaddr, sa, socklen);

  282.     ngx_queue_insert_tail(&bcf->groups, &grp->queue);

  284.     return grp;
  285. }


  288. static ngx_quic_sock_group_t *
  289. ‌ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
  290. {
  291.     int                     progfd, failed, flags, rc;
  292.     ngx_quic_bpf_conf_t    *bcf;
  293.     ngx_quic_sock_group_t  *grp;

  295.     bcf = ngx_quic_bpf_get_conf(cycle);

  297.     if (!bcf->enabled) {
  298.         return NULL;
  299.     }

  301.     grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen);
  302.     if (grp == NULL) {
  303.         return NULL;
  304.     }

  306.     grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH,
  307.                                      sizeof(uint64_t), sizeof(uint64_t),
  308.                                      bcf->map_size, 0);
  309.     if (grp->map_fd == -1) {
  310.         goto failed;
  311.     }

  313.     flags = fcntl(grp->map_fd, F_GETFD);
  314.     if (flags == -1) {
  315.         ngx_log_error(NGX_LOG_EMERG, cycle->log, errno,
  316.                       "quic bpf getfd failed");
  317.         goto failed;
  318.     }

  320.     /* need to inherit map during binary upgrade after exec */
  321.     flags &= ~FD_CLOEXEC;

  323.     rc = fcntl(grp->map_fd, F_SETFD, flags);
  324.     if (rc == -1) {
  325.         ngx_log_error(NGX_LOG_EMERG, cycle->log, errno,
  326.                       "quic bpf setfd failed");
  327.         goto failed;
  328.     }

  330.     ngx_bpf_program_link(&ngx_quic_reuseport_helper,
  331.                          "ngx_quic_sockmap", grp->map_fd);

  333.     progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper);
  334.     if (progfd < 0) {
  335.         goto failed;
  336.     }

  338.     failed = 0;

  340.     if (setsockopt(ls->fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF,
  341.                    &progfd, sizeof(int))
  342.         == -1)
  343.     {
  344.         ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
  345.                       "quic bpf setsockopt(SO_ATTACH_REUSEPORT_EBPF) failed");
  346.         failed = 1;
  347.     }

  349.     ngx_quic_bpf_close(cycle->log, progfd, "program");

  351.     if (failed) {
  352.         goto failed;
  353.     }

  355.     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
  356.                    "quic bpf sockmap created fd:%d", grp->map_fd);
  357.     return grp;

  359. failed:

  361.     if (grp->map_fd != -1) {
  362.         ngx_quic_bpf_close(cycle->log, grp->map_fd, "map");
  363.     }

  365.     ngx_queue_remove(&grp->queue);

  367.     return NULL;
  368. }


  371. static ngx_quic_sock_group_t *
  372. ‌ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
  373. {
  374.     ngx_quic_bpf_conf_t    *bcf, *old_bcf;
  375.     ngx_quic_sock_group_t  *grp, *ogrp;

  377.     bcf = ngx_quic_bpf_get_conf(cycle);

  379.     grp = ngx_quic_bpf_find_group(bcf, ls);
  380.     if (grp) {
  381.         return grp;
  382.     }

  384.     old_bcf = ngx_quic_bpf_get_old_conf(cycle);

  386.     if (old_bcf == NULL) {
  387.         return ngx_quic_bpf_create_group(cycle, ls);
  388.     }

  390.     ogrp = ngx_quic_bpf_find_group(old_bcf, ls);
  391.     if (ogrp == NULL) {
  392.         return ngx_quic_bpf_create_group(cycle, ls);
  393.     }

  395.     grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen);
  396.     if (grp == NULL) {
  397.         return NULL;
  398.     }

  400.     grp->map_fd = dup(ogrp->map_fd);
  401.     if (grp->map_fd == -1) {
  402.         ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
  403.                       "quic bpf failed to duplicate bpf map descriptor");

  405.         ngx_queue_remove(&grp->queue);

  407.         return NULL;
  408.     }

  410.     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
  411.                    "quic bpf sockmap fd duplicated old:%d new:%d",
  412.                    ogrp->map_fd, grp->map_fd);

  414.     return grp;
  415. }


  418. static ngx_int_t
  419. ‌ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle,  ngx_listening_t *ls)
  420. {
  421.     uint64_t                cookie;
  422.     ngx_quic_bpf_conf_t    *bcf;
  423.     ngx_quic_sock_group_t  *grp;

  425.     bcf = ngx_quic_bpf_get_conf(cycle);

  427.     grp = ngx_quic_bpf_get_group(cycle, ls);

  429.     if (grp == NULL) {
  430.         if (!bcf->enabled) {
  431.             return NGX_OK;
  432.         }

  434.         return NGX_ERROR;
  435.     }

  437.     grp->unused = 0;

  439.     cookie = ngx_quic_bpf_socket_key(ls->fd, cycle->log);
  440.     if (cookie == (uint64_t) NGX_ERROR) {
  441.         return NGX_ERROR;
  442.     }

  444.     /* map[cookie] = socket; for use in kernel helper */
  445.     if (ngx_bpf_map_update(grp->map_fd, &cookie, &ls->fd, BPF_ANY) == -1) {
  446.         ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
  447.                       "quic bpf failed to update socket map key=%xL", cookie);
  448.         return NGX_ERROR;
  449.     }

  451.     ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
  452.                  "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui",
  453.                  grp->map_fd, ls->fd, cookie, ls->worker);

  455.     /* do not inherit this socket */
  456.     ls->ignore = 1;

  458.     return NGX_OK;
  459. }


  462. static uint64_t
  463. ‌ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log)
  464. {
  465.     uint64_t   cookie;
  466.     socklen_t  optlen;

  468.     optlen = sizeof(cookie);

  470.     if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) {
  471.         ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
  472.                       "quic bpf getsockopt(SO_COOKIE) failed");

  474.         return (ngx_uint_t) NGX_ERROR;
  475.     }

  477.     return cookie;
  478. }


  481. static ngx_int_t
  482. ‌ngx_quic_bpf_export_maps(ngx_cycle_t *cycle)
  483. {
  484.     u_char                 *p, *buf;
  485.     size_t                  len;
  486.     ngx_str_t              *var;
  487.     ngx_queue_t            *q;
  488.     ngx_core_conf_t        *ccf;
  489.     ngx_quic_bpf_conf_t    *bcf;
  490.     ngx_quic_sock_group_t  *grp;

  492.     ccf = ngx_core_get_conf(cycle);
  493.     bcf = ngx_quic_bpf_get_conf(cycle);

  495.     len = sizeof(NGX_QUIC_BPF_VARNAME) + 1;

  497.     q = ngx_queue_head(&bcf->groups);

  499.     while (q != ngx_queue_sentinel(&bcf->groups)) {

  501.         grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);

  503.         q = ngx_queue_next(q);

  505.         if (grp->unused) {
  506.             /*
  507.              * map was inherited, but it is not used in this configuration;
  508.              * do not pass such map further and drop the group to prevent
  509.              * interference with changes during reload
  510.              */

  512.             ngx_quic_bpf_close(cycle->log, grp->map_fd, "map");
  513.             ngx_queue_remove(&grp->queue);

  515.             continue;
  516.         }

  518.         len += NGX_INT32_LEN + 1 + NGX_SOCKADDR_STRLEN + 1;
  519.     }

  521.     len++;

  523.     buf = ngx_palloc(cycle->pool, len);
  524.     if (buf == NULL) {
  525.         return NGX_ERROR;
  526.     }

  528.     p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=",
  529.                    sizeof(NGX_QUIC_BPF_VARNAME));

  531.     for (q = ngx_queue_head(&bcf->groups);
  532.          q != ngx_queue_sentinel(&bcf->groups);
  533.          q = ngx_queue_next(q))
  534.     {
  535.         grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);

  537.         p = ngx_sprintf(p, "%ud", grp->map_fd);

  539.         *p++ = NGX_QUIC_BPF_ADDRSEP;

  541.         p += ngx_sock_ntop(grp->sockaddr, grp->socklen, p,
  542.                            NGX_SOCKADDR_STRLEN, 1);

  544.         *p++ = NGX_QUIC_BPF_VARSEP;
  545.     }

  547.     *p = '\0';

  549.     var = ngx_array_push(&ccf->env);
  550.     if (var == NULL) {
  551.         return NGX_ERROR;
  552.     }

  554.     var->data = buf;
  555.     var->len = sizeof(NGX_QUIC_BPF_VARNAME) - 1;

  557.     return NGX_OK;
  558. }


  561. static ngx_int_t
  562. ‌ngx_quic_bpf_import_maps(ngx_cycle_t *cycle)
  563. {
  564.     int                     s;
  565.     u_char                 *inherited, *p, *v;
  566.     ngx_uint_t              in_fd;
  567.     ngx_addr_t              tmp;
  568.     ngx_quic_bpf_conf_t    *bcf;
  569.     ngx_quic_sock_group_t  *grp;

  571.     inherited = (u_char *) getenv(NGX_QUIC_BPF_VARNAME);

  573.     if (inherited == NULL) {
  574.         return NGX_OK;
  575.     }

  577.     bcf = ngx_quic_bpf_get_conf(cycle);

  579. #if (NGX_SUPPRESS_WARN)
  580.     s = -1;
  581. #endif

  583.     in_fd = 1;

  585.     for (p = inherited, v = p; *p; p++) {

  587.         switch (*p) {

  589.         case NGX_QUIC_BPF_ADDRSEP:

  591.             if (!in_fd) {
  592.                 ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
  593.                               "quic bpf failed to parse inherited env");
  594.                 return NGX_ERROR;
  595.             }
  596.             in_fd = 0;

  598.             s = ngx_atoi(v, p - v);
  599.             if (s == NGX_ERROR) {
  600.                 ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
  601.                               "quic bpf failed to parse inherited map fd");
  602.                 return NGX_ERROR;
  603.             }

  605.             v = p + 1;
  606.             break;

  608.         case NGX_QUIC_BPF_VARSEP:

  610.             if (in_fd) {
  611.                 ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
  612.                               "quic bpf failed to parse inherited env");
  613.                 return NGX_ERROR;
  614.             }
  615.             in_fd = 1;

  617.             grp = ngx_pcalloc(cycle->pool,
  618.                               sizeof(ngx_quic_sock_group_t));
  619.             if (grp == NULL) {
  620.                 return NGX_ERROR;
  621.             }

  623.             grp->map_fd = s;

  625.             if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v)
  626.                 != NGX_OK)
  627.             {
  628.                 ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
  629.                               "quic bpf failed to parse inherited"
  630.                               " address '%*s'", p - v , v);

  632.                 ngx_quic_bpf_close(cycle->log, s, "inherited map");

  634.                 return NGX_ERROR;
  635.             }

  637.             grp->sockaddr = tmp.sockaddr;
  638.             grp->socklen = tmp.socklen;

  640.             grp->unused = 1;

  642.             ngx_queue_insert_tail(&bcf->groups, &grp->queue);

  644.             ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
  645.                            "quic bpf sockmap inherited with "
  646.                            "fd:%d address:%*s",
  647.                            grp->map_fd, p - v, v);
  648.             v = p + 1;
  649.             break;

  651.         default:
  652.             break;
  653.         }
  654.     }

  656.     return NGX_OK;
  657. }

One Level Up Top Level