DPDK  20.05.0-rc0
rte_mempool.h
Go to the documentation of this file.
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2016 6WIND S.A.
4  */
5 
6 #ifndef _RTE_MEMPOOL_H_
7 #define _RTE_MEMPOOL_H_
8 
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <stdint.h>
39 #include <errno.h>
40 #include <inttypes.h>
41 #include <sys/queue.h>
42 
43 #include <rte_config.h>
44 #include <rte_spinlock.h>
45 #include <rte_log.h>
46 #include <rte_debug.h>
47 #include <rte_lcore.h>
48 #include <rte_memory.h>
49 #include <rte_branch_prediction.h>
50 #include <rte_ring.h>
51 #include <rte_memcpy.h>
52 #include <rte_common.h>
53 
54 #ifdef __cplusplus
55 extern "C" {
56 #endif
57 
58 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
59 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
60 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
62 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
63 
66 struct rte_mempool_debug_stats {
67  uint64_t put_bulk;
68  uint64_t put_objs;
69  uint64_t get_success_bulk;
70  uint64_t get_success_objs;
71  uint64_t get_fail_bulk;
72  uint64_t get_fail_objs;
74  uint64_t get_success_blks;
76  uint64_t get_fail_blks;
78 #endif
79 
84  uint32_t size;
85  uint32_t flushthresh;
86  uint32_t len;
87  /*
88  * Cache is allocated to this size to allow it to overflow in certain
89  * cases to avoid needless emptying of cache.
90  */
91  void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
93 
98  uint32_t elt_size;
99  uint32_t header_size;
100  uint32_t trailer_size;
101  uint32_t total_size;
103 };
104 
106 #define RTE_MEMPOOL_NAMESIZE (RTE_RING_NAMESIZE - \
107  sizeof(RTE_MEMPOOL_MZ_PREFIX) + 1)
108 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
109 
110 /* "MP_<name>" */
111 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
112 
113 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
114 
116 #define MEMPOOL_PG_NUM_DEFAULT 1
117 
118 #ifndef RTE_MEMPOOL_ALIGN
119 
122 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
123 #endif
124 
125 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
126 
137  STAILQ_ENTRY(rte_mempool_objhdr) next;
138  struct rte_mempool *mp;
140  union {
143  };
144 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
145  uint64_t cookie;
146 #endif
147 };
148 
152 STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr);
153 
154 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
155 
162 struct rte_mempool_objtlr {
163  uint64_t cookie;
164 };
165 
166 #endif
167 
171 STAILQ_HEAD(rte_mempool_memhdr_list, rte_mempool_memhdr);
172 
177  void *opaque);
178 
186  STAILQ_ENTRY(rte_mempool_memhdr) next;
187  struct rte_mempool *mp;
188  void *addr;
190  union {
193  };
194  size_t len;
196  void *opaque;
197 };
198 
210  unsigned int contig_block_size;
212 
216 struct rte_mempool {
217  /*
218  * Note: this field kept the RTE_MEMZONE_NAMESIZE size due to ABI
219  * compatibility requirements, it could be changed to
220  * RTE_MEMPOOL_NAMESIZE next time the ABI changes
221  */
222  char name[RTE_MEMZONE_NAMESIZE];
224  union {
225  void *pool_data;
226  uint64_t pool_id;
227  };
228  void *pool_config;
229  const struct rte_memzone *mz;
230  unsigned int flags;
231  int socket_id;
232  uint32_t size;
233  uint32_t cache_size;
236  uint32_t elt_size;
237  uint32_t header_size;
238  uint32_t trailer_size;
240  unsigned private_data_size;
248  int32_t ops_index;
249 
252  uint32_t populated_size;
253  struct rte_mempool_objhdr_list elt_list;
254  uint32_t nb_mem_chunks;
255  struct rte_mempool_memhdr_list mem_list;
257 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
258 
259  struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
260 #endif
262 
263 #define MEMPOOL_F_NO_SPREAD 0x0001
264 
265 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
266 #define MEMPOOL_F_SP_PUT 0x0004
267 #define MEMPOOL_F_SC_GET 0x0008
268 #define MEMPOOL_F_POOL_CREATED 0x0010
269 #define MEMPOOL_F_NO_IOVA_CONTIG 0x0020
270 #define MEMPOOL_F_NO_PHYS_CONTIG MEMPOOL_F_NO_IOVA_CONTIG /* deprecated */
271 
282 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
283 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
284  unsigned __lcore_id = rte_lcore_id(); \
285  if (__lcore_id < RTE_MAX_LCORE) { \
286  mp->stats[__lcore_id].name##_objs += n; \
287  mp->stats[__lcore_id].name##_bulk += 1; \
288  } \
289  } while(0)
290 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do { \
291  unsigned int __lcore_id = rte_lcore_id(); \
292  if (__lcore_id < RTE_MAX_LCORE) { \
293  mp->stats[__lcore_id].name##_blks += n; \
294  mp->stats[__lcore_id].name##_bulk += 1; \
295  } \
296  } while (0)
297 #else
298 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
299 #define __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, name, n) do {} while (0)
300 #endif
301 
310 #define MEMPOOL_HEADER_SIZE(mp, cs) \
311  (sizeof(*(mp)) + (((cs) == 0) ? 0 : \
312  (sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
313 
314 /* return the header of a mempool object (internal) */
315 static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
316 {
317  return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
318  sizeof(struct rte_mempool_objhdr));
319 }
320 
330 static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
331 {
332  struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
333  return hdr->mp;
334 }
335 
336 /* return the trailer of a mempool object (internal) */
337 static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
338 {
339  struct rte_mempool *mp = rte_mempool_from_obj(obj);
340  return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
341 }
342 
357 void rte_mempool_check_cookies(const struct rte_mempool *mp,
358  void * const *obj_table_const, unsigned n, int free);
359 
360 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
361 #define __mempool_check_cookies(mp, obj_table_const, n, free) \
362  rte_mempool_check_cookies(mp, obj_table_const, n, free)
363 #else
364 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
365 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
366 
386  void * const *first_obj_table_const, unsigned int n, int free);
387 
388 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
389 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
390  free) \
391  rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
392  free)
393 #else
394 #define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
395  free) \
396  do {} while (0)
397 #endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
398 
399 #define RTE_MEMPOOL_OPS_NAMESIZE 32
411 typedef int (*rte_mempool_alloc_t)(struct rte_mempool *mp);
412 
416 typedef void (*rte_mempool_free_t)(struct rte_mempool *mp);
417 
421 typedef int (*rte_mempool_enqueue_t)(struct rte_mempool *mp,
422  void * const *obj_table, unsigned int n);
423 
427 typedef int (*rte_mempool_dequeue_t)(struct rte_mempool *mp,
428  void **obj_table, unsigned int n);
429 
437  void **first_obj_table, unsigned int n);
438 
442 typedef unsigned (*rte_mempool_get_count)(const struct rte_mempool *mp);
443 
467 typedef ssize_t (*rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp,
468  uint32_t obj_num, uint32_t pg_shift,
469  size_t *min_chunk_size, size_t *align);
470 
509 __rte_experimental
510 ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp,
511  uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve,
512  size_t *min_chunk_size, size_t *align);
513 
521 ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp,
522  uint32_t obj_num, uint32_t pg_shift,
523  size_t *min_chunk_size, size_t *align);
524 
537 typedef void (rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp,
538  void *opaque, void *vaddr, rte_iova_t iova);
539 
568 typedef int (*rte_mempool_populate_t)(struct rte_mempool *mp,
569  unsigned int max_objs,
570  void *vaddr, rte_iova_t iova, size_t len,
571  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
572 
576 #define RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ 0x0001
577 
613 __rte_experimental
615  unsigned int flags, unsigned int max_objs,
616  void *vaddr, rte_iova_t iova, size_t len,
617  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
618 
626  unsigned int max_objs,
627  void *vaddr, rte_iova_t iova, size_t len,
628  rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg);
629 
636 typedef int (*rte_mempool_get_info_t)(const struct rte_mempool *mp,
637  struct rte_mempool_info *info);
638 
639 
667 
668 #define RTE_MEMPOOL_MAX_OPS_IDX 16
679 struct rte_mempool_ops_table {
681  uint32_t num_ops;
687 
690 
700 static inline struct rte_mempool_ops *
701 rte_mempool_get_ops(int ops_index)
702 {
703  RTE_VERIFY((ops_index >= 0) && (ops_index < RTE_MEMPOOL_MAX_OPS_IDX));
704 
705  return &rte_mempool_ops_table.ops[ops_index];
706 }
707 
717 int
718 rte_mempool_ops_alloc(struct rte_mempool *mp);
719 
733 static inline int
734 rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
735  void **obj_table, unsigned n)
736 {
737  struct rte_mempool_ops *ops;
738 
739  ops = rte_mempool_get_ops(mp->ops_index);
740  return ops->dequeue(mp, obj_table, n);
741 }
742 
756 static inline int
757 rte_mempool_ops_dequeue_contig_blocks(struct rte_mempool *mp,
758  void **first_obj_table, unsigned int n)
759 {
760  struct rte_mempool_ops *ops;
761 
762  ops = rte_mempool_get_ops(mp->ops_index);
763  RTE_ASSERT(ops->dequeue_contig_blocks != NULL);
764  return ops->dequeue_contig_blocks(mp, first_obj_table, n);
765 }
766 
780 static inline int
781 rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
782  unsigned n)
783 {
784  struct rte_mempool_ops *ops;
785 
786  ops = rte_mempool_get_ops(mp->ops_index);
787  return ops->enqueue(mp, obj_table, n);
788 }
789 
798 unsigned
799 rte_mempool_ops_get_count(const struct rte_mempool *mp);
800 
820 ssize_t rte_mempool_ops_calc_mem_size(const struct rte_mempool *mp,
821  uint32_t obj_num, uint32_t pg_shift,
822  size_t *min_chunk_size, size_t *align);
823 
847 int rte_mempool_ops_populate(struct rte_mempool *mp, unsigned int max_objs,
848  void *vaddr, rte_iova_t iova, size_t len,
850  void *obj_cb_arg);
851 
867 __rte_experimental
868 int rte_mempool_ops_get_info(const struct rte_mempool *mp,
869  struct rte_mempool_info *info);
870 
877 void
878 rte_mempool_ops_free(struct rte_mempool *mp);
879 
897 int
898 rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
899  void *pool_config);
900 
911 int rte_mempool_register_ops(const struct rte_mempool_ops *ops);
912 
918 #define MEMPOOL_REGISTER_OPS(ops) \
919  RTE_INIT(mp_hdlr_init_##ops) \
920  { \
921  rte_mempool_register_ops(&ops); \
922  }
923 
929 typedef void (rte_mempool_obj_cb_t)(struct rte_mempool *mp,
930  void *opaque, void *obj, unsigned obj_idx);
931 typedef rte_mempool_obj_cb_t rte_mempool_obj_ctor_t; /* compat */
932 
938 typedef void (rte_mempool_mem_cb_t)(struct rte_mempool *mp,
939  void *opaque, struct rte_mempool_memhdr *memhdr,
940  unsigned mem_idx);
941 
948 typedef void (rte_mempool_ctor_t)(struct rte_mempool *, void *);
949 
1029 struct rte_mempool *
1030 rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
1031  unsigned cache_size, unsigned private_data_size,
1032  rte_mempool_ctor_t *mp_init, void *mp_init_arg,
1033  rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
1034  int socket_id, unsigned flags);
1035 
1070 struct rte_mempool *
1071 rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
1072  unsigned cache_size, unsigned private_data_size,
1073  int socket_id, unsigned flags);
1084 void
1085 rte_mempool_free(struct rte_mempool *mp);
1086 
1114 int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
1115  rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
1116  void *opaque);
1117 
1141 int
1142 rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
1143  size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
1144  void *opaque);
1145 
1160 
1174 int rte_mempool_populate_anon(struct rte_mempool *mp);
1175 
1191 uint32_t rte_mempool_obj_iter(struct rte_mempool *mp,
1192  rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg);
1193 
1209 uint32_t rte_mempool_mem_iter(struct rte_mempool *mp,
1210  rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg);
1211 
1220 void rte_mempool_dump(FILE *f, struct rte_mempool *mp);
1221 
1236 struct rte_mempool_cache *
1237 rte_mempool_cache_create(uint32_t size, int socket_id);
1238 
1245 void
1247 
1258 static __rte_always_inline struct rte_mempool_cache *
1259 rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
1260 {
1261  if (mp->cache_size == 0)
1262  return NULL;
1263 
1264  if (lcore_id >= RTE_MAX_LCORE)
1265  return NULL;
1266 
1267  return &mp->local_cache[lcore_id];
1268 }
1269 
1278 static __rte_always_inline void
1280  struct rte_mempool *mp)
1281 {
1282  if (cache == NULL)
1283  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1284  if (cache == NULL || cache->len == 0)
1285  return;
1286  rte_mempool_ops_enqueue_bulk(mp, cache->objs, cache->len);
1287  cache->len = 0;
1288 }
1289 
1302 static __rte_always_inline void
1303 __mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1304  unsigned int n, struct rte_mempool_cache *cache)
1305 {
1306  void **cache_objs;
1307 
1308  /* increment stat now, adding in mempool always success */
1309  __MEMPOOL_STAT_ADD(mp, put, n);
1310 
1311  /* No cache provided or if put would overflow mem allocated for cache */
1312  if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
1313  goto ring_enqueue;
1314 
1315  cache_objs = &cache->objs[cache->len];
1316 
1317  /*
1318  * The cache follows the following algorithm
1319  * 1. Add the objects to the cache
1320  * 2. Anything greater than the cache min value (if it crosses the
1321  * cache flush threshold) is flushed to the ring.
1322  */
1323 
1324  /* Add elements back into the cache */
1325  rte_memcpy(&cache_objs[0], obj_table, sizeof(void *) * n);
1326 
1327  cache->len += n;
1328 
1329  if (cache->len >= cache->flushthresh) {
1330  rte_mempool_ops_enqueue_bulk(mp, &cache->objs[cache->size],
1331  cache->len - cache->size);
1332  cache->len = cache->size;
1333  }
1334 
1335  return;
1336 
1337 ring_enqueue:
1338 
1339  /* push remaining objects in ring */
1340 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1341  if (rte_mempool_ops_enqueue_bulk(mp, obj_table, n) < 0)
1342  rte_panic("cannot put objects in mempool\n");
1343 #else
1344  rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
1345 #endif
1346 }
1347 
1348 
1361 static __rte_always_inline void
1362 rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
1363  unsigned int n, struct rte_mempool_cache *cache)
1364 {
1365  __mempool_check_cookies(mp, obj_table, n, 0);
1366  __mempool_generic_put(mp, obj_table, n, cache);
1367 }
1368 
1383 static __rte_always_inline void
1384 rte_mempool_put_bulk(struct rte_mempool *mp, void * const *obj_table,
1385  unsigned int n)
1386 {
1387  struct rte_mempool_cache *cache;
1388  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1389  rte_mempool_generic_put(mp, obj_table, n, cache);
1390 }
1391 
1404 static __rte_always_inline void
1405 rte_mempool_put(struct rte_mempool *mp, void *obj)
1406 {
1407  rte_mempool_put_bulk(mp, &obj, 1);
1408 }
1409 
1424 static __rte_always_inline int
1425 __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1426  unsigned int n, struct rte_mempool_cache *cache)
1427 {
1428  int ret;
1429  uint32_t index, len;
1430  void **cache_objs;
1431 
1432  /* No cache provided or cannot be satisfied from cache */
1433  if (unlikely(cache == NULL || n >= cache->size))
1434  goto ring_dequeue;
1435 
1436  cache_objs = cache->objs;
1437 
1438  /* Can this be satisfied from the cache? */
1439  if (cache->len < n) {
1440  /* No. Backfill the cache first, and then fill from it */
1441  uint32_t req = n + (cache->size - cache->len);
1442 
1443  /* How many do we require i.e. number to fill the cache + the request */
1444  ret = rte_mempool_ops_dequeue_bulk(mp,
1445  &cache->objs[cache->len], req);
1446  if (unlikely(ret < 0)) {
1447  /*
1448  * In the off chance that we are buffer constrained,
1449  * where we are not able to allocate cache + n, go to
1450  * the ring directly. If that fails, we are truly out of
1451  * buffers.
1452  */
1453  goto ring_dequeue;
1454  }
1455 
1456  cache->len += req;
1457  }
1458 
1459  /* Now fill in the response ... */
1460  for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
1461  *obj_table = cache_objs[len];
1462 
1463  cache->len -= n;
1464 
1465  __MEMPOOL_STAT_ADD(mp, get_success, n);
1466 
1467  return 0;
1468 
1469 ring_dequeue:
1470 
1471  /* get remaining objects from ring */
1472  ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
1473 
1474  if (ret < 0)
1475  __MEMPOOL_STAT_ADD(mp, get_fail, n);
1476  else
1477  __MEMPOOL_STAT_ADD(mp, get_success, n);
1478 
1479  return ret;
1480 }
1481 
1502 static __rte_always_inline int
1503 rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
1504  unsigned int n, struct rte_mempool_cache *cache)
1505 {
1506  int ret;
1507  ret = __mempool_generic_get(mp, obj_table, n, cache);
1508  if (ret == 0)
1509  __mempool_check_cookies(mp, obj_table, n, 1);
1510  return ret;
1511 }
1512 
1535 static __rte_always_inline int
1536 rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
1537 {
1538  struct rte_mempool_cache *cache;
1539  cache = rte_mempool_default_cache(mp, rte_lcore_id());
1540  return rte_mempool_generic_get(mp, obj_table, n, cache);
1541 }
1542 
1563 static __rte_always_inline int
1564 rte_mempool_get(struct rte_mempool *mp, void **obj_p)
1565 {
1566  return rte_mempool_get_bulk(mp, obj_p, 1);
1567 }
1568 
1593 static __rte_always_inline int
1594 __rte_experimental
1596  void **first_obj_table, unsigned int n)
1597 {
1598  int ret;
1599 
1600  ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
1601  if (ret == 0) {
1602  __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_success, n);
1603  __mempool_contig_blocks_check_cookies(mp, first_obj_table, n,
1604  1);
1605  } else {
1606  __MEMPOOL_CONTIG_BLOCKS_STAT_ADD(mp, get_fail, n);
1607  }
1608 
1609  return ret;
1610 }
1611 
1624 unsigned int rte_mempool_avail_count(const struct rte_mempool *mp);
1625 
1638 unsigned int
1639 rte_mempool_in_use_count(const struct rte_mempool *mp);
1640 
1654 static inline int
1656 {
1657  return rte_mempool_avail_count(mp) == mp->size;
1658 }
1659 
1673 static inline int
1675 {
1676  return rte_mempool_avail_count(mp) == 0;
1677 }
1678 
1689 static inline rte_iova_t
1690 rte_mempool_virt2iova(const void *elt)
1691 {
1692  const struct rte_mempool_objhdr *hdr;
1693  hdr = (const struct rte_mempool_objhdr *)RTE_PTR_SUB(elt,
1694  sizeof(*hdr));
1695  return hdr->iova;
1696 }
1697 
1708 void rte_mempool_audit(struct rte_mempool *mp);
1709 
1718 static inline void *rte_mempool_get_priv(struct rte_mempool *mp)
1719 {
1720  return (char *)mp +
1722 }
1723 
1730 void rte_mempool_list_dump(FILE *f);
1731 
1744 struct rte_mempool *rte_mempool_lookup(const char *name);
1745 
1763 uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
1764  struct rte_mempool_objsz *sz);
1765 
1774 void rte_mempool_walk(void (*func)(struct rte_mempool *, void *arg),
1775  void *arg);
1776 
1784 __rte_experimental
1785 int
1786 rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz);
1787 
1788 #ifdef __cplusplus
1789 }
1790 #endif
1791 
1792 #endif /* _RTE_MEMPOOL_H_ */
__rte_experimental int rte_mempool_op_populate_helper(struct rte_mempool *mp, unsigned int flags, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
uint64_t pool_id
Definition: rte_mempool.h:226
#define __rte_always_inline
Definition: rte_common.h:183
struct rte_mempool * rte_mempool_lookup(const char *name)
struct rte_mempool_cache * rte_mempool_cache_create(uint32_t size, int socket_id)
uint32_t rte_mempool_mem_iter(struct rte_mempool *mp, rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
uint32_t header_size
Definition: rte_mempool.h:237
const struct rte_memzone * mz
Definition: rte_mempool.h:229
uint64_t rte_iova_t
Definition: rte_common.h:365
static __rte_always_inline void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:1384
struct rte_mempool * mp
Definition: rte_mempool.h:187
void rte_mempool_list_dump(FILE *f)
int(* rte_mempool_dequeue_contig_blocks_t)(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:436
static __rte_always_inline int __rte_experimental rte_mempool_get_contig_blocks(struct rte_mempool *mp, void **first_obj_table, unsigned int n)
Definition: rte_mempool.h:1595
rte_mempool_alloc_t alloc
Definition: rte_mempool.h:643
void( rte_mempool_memchunk_free_cb_t)(struct rte_mempool_memhdr *memhdr, void *opaque)
Definition: rte_mempool.h:176
unsigned int flags
Definition: rte_mempool.h:230
void( rte_mempool_obj_cb_t)(struct rte_mempool *mp, void *opaque, void *obj, unsigned obj_idx)
Definition: rte_mempool.h:929
static __rte_always_inline void rte_mempool_generic_put(struct rte_mempool *mp, void *const *obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1362
int(* rte_mempool_alloc_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:411
static int rte_mempool_empty(const struct rte_mempool *mp)
Definition: rte_mempool.h:1674
phys_addr_t physaddr
Definition: rte_mempool.h:142
phys_addr_t phys_addr
Definition: rte_mempool.h:192
rte_mempool_memchunk_free_cb_t * free_cb
Definition: rte_mempool.h:195
char name[RTE_MEMPOOL_OPS_NAMESIZE]
Definition: rte_mempool.h:642
int(* rte_mempool_dequeue_t)(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:427
#define MEMPOOL_HEADER_SIZE(mp, cs)
Definition: rte_mempool.h:310
rte_mempool_get_info_t get_info
Definition: rte_mempool.h:661
rte_mempool_dequeue_contig_blocks_t dequeue_contig_blocks
Definition: rte_mempool.h:665
uint32_t cache_size
Definition: rte_mempool.h:233
unsigned int rte_mempool_avail_count(const struct rte_mempool *mp)
uint32_t size
Definition: rte_mempool.h:232
#define RTE_PTR_ADD(ptr, x)
Definition: rte_common.h:195
void rte_mempool_cache_free(struct rte_mempool_cache *cache)
uint32_t total_size
Definition: rte_mempool.h:101
uint32_t rte_mempool_obj_iter(struct rte_mempool *mp, rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
int rte_mempool_register_ops(const struct rte_mempool_ops *ops)
#define unlikely(x)
static __rte_always_inline int rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table, unsigned int n, struct rte_mempool_cache *cache)
Definition: rte_mempool.h:1503
int rte_mempool_populate_default(struct rte_mempool *mp)
static __rte_always_inline void rte_mempool_cache_flush(struct rte_mempool_cache *cache, struct rte_mempool *mp)
Definition: rte_mempool.h:1279
#define RTE_MEMPOOL_OPS_NAMESIZE
Definition: rte_mempool.h:399
static __rte_always_inline int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
Definition: rte_mempool.h:1564
STAILQ_HEAD(rte_mempool_objhdr_list, rte_mempool_objhdr)
uint32_t nb_mem_chunks
Definition: rte_mempool.h:254
uint64_t phys_addr_t
Definition: rte_common.h:355
struct rte_mempool * mp
Definition: rte_mempool.h:138
struct rte_mempool_ops ops[RTE_MEMPOOL_MAX_OPS_IDX]
Definition: rte_mempool.h:685
rte_mempool_get_count get_count
Definition: rte_mempool.h:647
ssize_t(* rte_mempool_calc_mem_size_t)(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
Definition: rte_mempool.h:467
static int rte_mempool_full(const struct rte_mempool *mp)
Definition: rte_mempool.h:1655
static unsigned rte_lcore_id(void)
Definition: rte_lcore.h:51
void * pool_config
Definition: rte_mempool.h:228
unsigned int rte_mempool_in_use_count(const struct rte_mempool *mp)
void( rte_mempool_mem_cb_t)(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned mem_idx)
Definition: rte_mempool.h:938
static __rte_always_inline int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned int n)
Definition: rte_mempool.h:1536
#define rte_panic(...)
Definition: rte_debug.h:50
int rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name, void *pool_config)
__rte_experimental int rte_mempool_ops_get_info(const struct rte_mempool *mp, struct rte_mempool_info *info)
int rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
rte_mempool_calc_mem_size_t calc_mem_size
Definition: rte_mempool.h:652
unsigned(* rte_mempool_get_count)(const struct rte_mempool *mp)
Definition: rte_mempool.h:442
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
ssize_t rte_mempool_op_calc_mem_size_default(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
void * objs[RTE_MEMPOOL_CACHE_MAX_SIZE *3]
Definition: rte_mempool.h:91
void * pool_data
Definition: rte_mempool.h:225
unsigned int contig_block_size
Definition: rte_mempool.h:210
int rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb, void *opaque)
int(* rte_mempool_populate_t)(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
Definition: rte_mempool.h:568
uint32_t elt_size
Definition: rte_mempool.h:236
void rte_mempool_audit(struct rte_mempool *mp)
int(* rte_mempool_get_info_t)(const struct rte_mempool *mp, struct rte_mempool_info *info)
Definition: rte_mempool.h:636
struct rte_mempool * rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, int socket_id, unsigned flags)
unsigned private_data_size
Definition: rte_mempool.h:240
void( rte_mempool_ctor_t)(struct rte_mempool *, void *)
Definition: rte_mempool.h:948
__rte_experimental int rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz)
rte_spinlock_t sl
Definition: rte_mempool.h:680
static __rte_always_inline struct rte_mempool_cache * rte_mempool_default_cache(struct rte_mempool *mp, unsigned lcore_id)
Definition: rte_mempool.h:1259
void rte_mempool_dump(FILE *f, struct rte_mempool *mp)
int rte_mempool_populate_anon(struct rte_mempool *mp)
struct rte_mempool_cache * local_cache
Definition: rte_mempool.h:250
uint32_t trailer_size
Definition: rte_mempool.h:238
#define RTE_STD_C11
Definition: rte_common.h:40
rte_mempool_dequeue_t dequeue
Definition: rte_mempool.h:646
int(* rte_mempool_enqueue_t)(struct rte_mempool *mp, void *const *obj_table, unsigned int n)
Definition: rte_mempool.h:421
void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp, void *const *first_obj_table_const, unsigned int n, int free)
#define __rte_cache_aligned
Definition: rte_common.h:347
uint32_t flushthresh
Definition: rte_mempool.h:85
__rte_experimental ssize_t rte_mempool_op_calc_mem_size_helper(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t chunk_reserve, size_t *min_chunk_size, size_t *align)
int rte_mempool_op_populate_default(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, rte_iova_t iova, size_t len, rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_cb_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_PTR_SUB(ptr, x)
Definition: rte_common.h:200
uint32_t elt_size
Definition: rte_mempool.h:98
static struct rte_mempool * rte_mempool_from_obj(void *obj)
Definition: rte_mempool.h:330
uint32_t header_size
Definition: rte_mempool.h:99
int32_t ops_index
Definition: rte_mempool.h:248
static void * rte_memcpy(void *dst, const void *src, size_t n)
uint32_t populated_size
Definition: rte_mempool.h:252
static rte_iova_t rte_mempool_virt2iova(const void *elt)
Definition: rte_mempool.h:1690
static __rte_always_inline void rte_mempool_put(struct rte_mempool *mp, void *obj)
Definition: rte_mempool.h:1405
rte_mempool_populate_t populate
Definition: rte_mempool.h:657
void( rte_mempool_populate_obj_cb_t)(struct rte_mempool *mp, void *opaque, void *vaddr, rte_iova_t iova)
Definition: rte_mempool.h:537
void rte_mempool_free(struct rte_mempool *mp)
void rte_mempool_walk(void(*func)(struct rte_mempool *, void *arg), void *arg)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
Definition: rte_mempool.h:1718
char name[RTE_MEMZONE_NAMESIZE]
Definition: rte_mempool.h:222
uint32_t trailer_size
Definition: rte_mempool.h:100
#define RTE_MEMPOOL_MAX_OPS_IDX
Definition: rte_mempool.h:668
void(* rte_mempool_free_t)(struct rte_mempool *mp)
Definition: rte_mempool.h:416
rte_mempool_free_t free
Definition: rte_mempool.h:644
rte_mempool_enqueue_t enqueue
Definition: rte_mempool.h:645
#define RTE_MEMZONE_NAMESIZE
Definition: rte_memzone.h:51