96 #include <sys/queue.h> 105 #define RTE_TAILQ_RING_NAME "RTE_RING" 107 enum rte_ring_queue_behavior {
108 RTE_RING_QUEUE_FIXED = 0,
109 RTE_RING_QUEUE_VARIABLE
112 #define RTE_RING_MZ_PREFIX "RG_" 114 #define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \ 115 sizeof(RTE_RING_MZ_PREFIX) + 1) 119 #if RTE_CACHE_LINE_SIZE < 128 120 #define PROD_ALIGN (RTE_CACHE_LINE_SIZE * 2) 121 #define CONS_ALIGN (RTE_CACHE_LINE_SIZE * 2) 123 #define PROD_ALIGN RTE_CACHE_LINE_SIZE 124 #define CONS_ALIGN RTE_CACHE_LINE_SIZE 128 struct rte_ring_headtail {
129 volatile uint32_t head;
130 volatile uint32_t tail;
164 #define RING_F_SP_ENQ 0x0001 165 #define RING_F_SC_DEQ 0x0002 166 #define RTE_RING_SZ_MASK (unsigned)(0x0fffffff) 267 int socket_id,
unsigned flags);
289 #define ENQUEUE_PTRS(r, ring_start, prod_head, obj_table, n, obj_type) do { \ 291 const uint32_t size = (r)->size; \ 292 uint32_t idx = prod_head & (r)->mask; \ 293 obj_type *ring = (obj_type *)ring_start; \ 294 if (likely(idx + n < size)) { \ 295 for (i = 0; i < (n & ((~(unsigned)0x3))); i+=4, idx+=4) { \ 296 ring[idx] = obj_table[i]; \ 297 ring[idx+1] = obj_table[i+1]; \ 298 ring[idx+2] = obj_table[i+2]; \ 299 ring[idx+3] = obj_table[i+3]; \ 303 ring[idx++] = obj_table[i++]; \ 305 ring[idx++] = obj_table[i++]; \ 307 ring[idx++] = obj_table[i++]; \ 310 for (i = 0; idx < size; i++, idx++)\ 311 ring[idx] = obj_table[i]; \ 312 for (idx = 0; i < n; i++, idx++) \ 313 ring[idx] = obj_table[i]; \ 320 #define DEQUEUE_PTRS(r, ring_start, cons_head, obj_table, n, obj_type) do { \ 322 uint32_t idx = cons_head & (r)->mask; \ 323 const uint32_t size = (r)->size; \ 324 obj_type *ring = (obj_type *)ring_start; \ 325 if (likely(idx + n < size)) { \ 326 for (i = 0; i < (n & (~(unsigned)0x3)); i+=4, idx+=4) {\ 327 obj_table[i] = ring[idx]; \ 328 obj_table[i+1] = ring[idx+1]; \ 329 obj_table[i+2] = ring[idx+2]; \ 330 obj_table[i+3] = ring[idx+3]; \ 334 obj_table[i++] = ring[idx++]; \ 336 obj_table[i++] = ring[idx++]; \ 338 obj_table[i++] = ring[idx++]; \ 341 for (i = 0; idx < size; i++, idx++) \ 342 obj_table[i] = ring[idx]; \ 343 for (idx = 0; i < n; i++, idx++) \ 344 obj_table[i] = ring[idx]; \ 348 static inline __attribute__((always_inline)) void
349 update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val,
357 while (
unlikely(ht->tail != old_val))
386 static inline __attribute__((always_inline)) unsigned
int 387 __rte_ring_move_prod_head(struct
rte_ring *r,
int is_sp,
388 unsigned int n, enum rte_ring_queue_behavior behavior,
389 uint32_t *old_head, uint32_t *new_head,
390 uint32_t *free_entries)
392 const uint32_t
mask = r->mask;
393 unsigned int max = n;
400 *old_head = r->prod.head;
401 const uint32_t cons_tail = r->cons.tail;
406 *free_entries = (mask + cons_tail - *old_head);
410 n = (behavior == RTE_RING_QUEUE_FIXED) ?
416 *new_head = *old_head + n;
418 r->prod.head = *new_head, success = 1;
421 *old_head, *new_head);
446 static inline __attribute__((always_inline)) unsigned
int 447 __rte_ring_do_enqueue(struct
rte_ring *r,
void * const *obj_table,
448 unsigned int n, enum rte_ring_queue_behavior behavior,
449 int is_sp,
unsigned int *free_space)
451 uint32_t prod_head, prod_next;
452 uint32_t free_entries;
454 n = __rte_ring_move_prod_head(r, is_sp, n, behavior,
455 &prod_head, &prod_next, &free_entries);
459 ENQUEUE_PTRS(r, &r[1], prod_head, obj_table, n,
void *);
462 update_tail(&r->prod, prod_head, prod_next, is_sp);
464 if (free_space != NULL)
465 *free_space = free_entries - n;
492 static inline __attribute__((always_inline)) unsigned
int 493 __rte_ring_move_cons_head(struct
rte_ring *r,
int is_sc,
494 unsigned int n, enum rte_ring_queue_behavior behavior,
495 uint32_t *old_head, uint32_t *new_head,
498 unsigned int max = n;
506 *old_head = r->cons.head;
507 const uint32_t prod_tail = r->prod.tail;
512 *entries = (prod_tail - *old_head);
516 n = (behavior == RTE_RING_QUEUE_FIXED) ? 0 : *entries;
521 *new_head = *old_head + n;
523 r->cons.head = *new_head, success = 1;
551 static inline __attribute__((always_inline)) unsigned
int 552 __rte_ring_do_dequeue(struct
rte_ring *r,
void **obj_table,
553 unsigned int n, enum rte_ring_queue_behavior behavior,
554 int is_sc,
unsigned int *available)
556 uint32_t cons_head, cons_next;
559 n = __rte_ring_move_cons_head(r, is_sc, n, behavior,
560 &cons_head, &cons_next, &entries);
564 DEQUEUE_PTRS(r, &r[1], cons_head, obj_table, n,
void *);
567 update_tail(&r->cons, cons_head, cons_next, is_sc);
570 if (available != NULL)
571 *available = entries - n;
593 static inline unsigned int __attribute__((always_inline))
595 unsigned int n,
unsigned int *free_space)
597 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
598 __IS_MP, free_space);
616 static inline unsigned int __attribute__((always_inline))
618 unsigned int n,
unsigned int *free_space)
620 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
621 __IS_SP, free_space);
643 static inline unsigned int __attribute__((always_inline))
645 unsigned int n,
unsigned int *free_space)
647 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
648 r->prod.single, free_space);
665 static inline int __attribute__((always_inline))
682 static inline int __attribute__((always_inline))
703 static inline int __attribute__((always_inline))
727 static inline unsigned int __attribute__((always_inline))
729 unsigned int n,
unsigned int *available)
731 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
751 static inline unsigned int __attribute__((always_inline))
753 unsigned int n,
unsigned int *available)
755 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
778 static inline unsigned int __attribute__((always_inline))
780 unsigned int *available)
782 return __rte_ring_do_dequeue(r, obj_table, n, RTE_RING_QUEUE_FIXED,
783 r->cons.single, available);
801 static inline int __attribute__((always_inline))
819 static inline int __attribute__((always_inline))
841 static inline int __attribute__((always_inline))
859 uint32_t prod_tail = r->prod.tail;
860 uint32_t cons_tail = r->cons.tail;
861 return ((cons_tail - prod_tail - 1) & r->
mask) == 0;
876 uint32_t prod_tail = r->prod.tail;
877 uint32_t cons_tail = r->cons.tail;
878 return !!(cons_tail == prod_tail);
889 static inline unsigned 892 uint32_t prod_tail = r->prod.tail;
893 uint32_t cons_tail = r->cons.tail;
894 return (prod_tail - cons_tail) & r->
mask;
905 static inline unsigned 908 uint32_t prod_tail = r->prod.tail;
909 uint32_t cons_tail = r->cons.tail;
910 return (cons_tail - prod_tail - 1) & r->
mask;
921 static inline unsigned int 965 static inline unsigned __attribute__((always_inline))
967 unsigned int n,
unsigned int *free_space)
969 return __rte_ring_do_enqueue(r, obj_table, n,
970 RTE_RING_QUEUE_VARIABLE, __IS_MP, free_space);
988 static inline unsigned __attribute__((always_inline))
990 unsigned int n,
unsigned int *free_space)
992 return __rte_ring_do_enqueue(r, obj_table, n,
993 RTE_RING_QUEUE_VARIABLE, __IS_SP, free_space);
1015 static inline unsigned __attribute__((always_inline))
1017 unsigned int n,
unsigned int *free_space)
1019 return __rte_ring_do_enqueue(r, obj_table, n, RTE_RING_QUEUE_VARIABLE,
1020 r->prod.single, free_space);
1043 static inline unsigned __attribute__((always_inline))
1045 unsigned int n,
unsigned int *available)
1047 return __rte_ring_do_dequeue(r, obj_table, n,
1048 RTE_RING_QUEUE_VARIABLE, __IS_MC, available);
1068 static inline unsigned __attribute__((always_inline))
1070 unsigned int n,
unsigned int *available)
1072 return __rte_ring_do_dequeue(r, obj_table, n,
1073 RTE_RING_QUEUE_VARIABLE, __IS_SC, available);
1095 static inline unsigned __attribute__((always_inline))
1097 unsigned int n,
unsigned int *available)
1099 return __rte_ring_do_dequeue(r, obj_table, n,
1100 RTE_RING_QUEUE_VARIABLE,
1101 r->cons.single, available);
static unsigned int rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static unsigned rte_ring_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static void rte_smp_rmb(void)
static int rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
const struct rte_memzone * memzone
static int rte_ring_sc_dequeue(struct rte_ring *r, void **obj_p)
static unsigned rte_ring_sp_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static int rte_ring_dequeue(struct rte_ring *r, void **obj_p)
static unsigned rte_ring_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
char name[RTE_MEMZONE_NAMESIZE] __rte_cache_aligned
static unsigned rte_ring_mc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static unsigned rte_ring_sc_dequeue_burst(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static unsigned rte_ring_mp_enqueue_burst(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static int rte_ring_empty(const struct rte_ring *r)
static unsigned int rte_ring_sp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
void rte_ring_list_dump(FILE *f)
static unsigned int rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
static unsigned int rte_ring_get_size(const struct rte_ring *r)
void rte_ring_free(struct rte_ring *r)
static void rte_smp_wmb(void)
static int rte_ring_enqueue(struct rte_ring *r, void *obj)
static unsigned int rte_ring_mp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
static int rte_ring_sp_enqueue(struct rte_ring *r, void *obj)
void rte_ring_dump(FILE *f, const struct rte_ring *r)
static unsigned rte_ring_count(const struct rte_ring *r)
struct rte_ring * rte_ring_create(const char *name, unsigned count, int socket_id, unsigned flags)
static int rte_ring_mp_enqueue(struct rte_ring *r, void *obj)
static unsigned int rte_ring_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned int n, unsigned int *free_space)
struct rte_ring * rte_ring_lookup(const char *name)
static unsigned rte_ring_free_count(const struct rte_ring *r)
static int rte_ring_mc_dequeue(struct rte_ring *r, void **obj_p)
int rte_ring_init(struct rte_ring *r, const char *name, unsigned count, unsigned flags)
static int rte_ring_full(const struct rte_ring *r)
ssize_t rte_ring_get_memsize(unsigned count)
static unsigned int rte_ring_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned int n, unsigned int *available)
#define RTE_MEMZONE_NAMESIZE