2 * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
22 #include "openvswitch/list.h"
25 #include "netdev-dpdk.h"
31 enum OVS_PACKED_ENUM dp_packet_source {
32 DPBUF_MALLOC, /* Obtained via malloc(). */
33 DPBUF_STACK, /* Un-movable stack space or static buffer. */
34 DPBUF_STUB, /* Starts on stack, may expand into heap. */
35 DPBUF_DPDK, /* buffer data is from DPDK allocated memory.
36 * ref to build_dp_packet() in netdev-dpdk. */
39 #define DP_PACKET_CONTEXT_SIZE 64
41 /* Buffer for holding packet data. A dp_packet is automatically reallocated
42 * as necessary if it grows too large for the available memory.
46 struct rte_mbuf mbuf; /* DPDK mbuf */
48 void *base_; /* First byte of allocated space. */
49 uint16_t allocated_; /* Number of bytes allocated. */
50 uint16_t data_ofs; /* First byte actually in use. */
51 uint32_t size_; /* Number of bytes in use. */
52 uint32_t rss_hash; /* Packet hash. */
53 bool rss_hash_valid; /* Is the 'rss_hash' valid? */
55 enum dp_packet_source source; /* Source of memory allocated as 'base'. */
56 uint8_t l2_pad_size; /* Detected l2 padding size.
57 * Padding is non-pullable. */
58 uint16_t l2_5_ofs; /* MPLS label stack offset, or UINT16_MAX */
59 uint16_t l3_ofs; /* Network-level header offset,
61 uint16_t l4_ofs; /* Transport-level header offset,
63 uint32_t cutlen; /* length in bytes to cut from the end. */
65 struct pkt_metadata md;
66 uint64_t data[DP_PACKET_CONTEXT_SIZE / 8];
70 static inline void *dp_packet_data(const struct dp_packet *);
71 static inline void dp_packet_set_data(struct dp_packet *, void *);
72 static inline void *dp_packet_base(const struct dp_packet *);
73 static inline void dp_packet_set_base(struct dp_packet *, void *);
75 static inline uint32_t dp_packet_size(const struct dp_packet *);
76 static inline void dp_packet_set_size(struct dp_packet *, uint32_t);
78 static inline uint16_t dp_packet_get_allocated(const struct dp_packet *);
79 static inline void dp_packet_set_allocated(struct dp_packet *, uint16_t);
81 void *dp_packet_resize_l2(struct dp_packet *, int increment);
82 void *dp_packet_resize_l2_5(struct dp_packet *, int increment);
83 static inline void *dp_packet_l2(const struct dp_packet *);
84 static inline void dp_packet_reset_offsets(struct dp_packet *);
85 static inline uint8_t dp_packet_l2_pad_size(const struct dp_packet *);
86 static inline void dp_packet_set_l2_pad_size(struct dp_packet *, uint8_t);
87 static inline void *dp_packet_l2_5(const struct dp_packet *);
88 static inline void dp_packet_set_l2_5(struct dp_packet *, void *);
89 static inline void *dp_packet_l3(const struct dp_packet *);
90 static inline void dp_packet_set_l3(struct dp_packet *, void *);
91 static inline void *dp_packet_l4(const struct dp_packet *);
92 static inline void dp_packet_set_l4(struct dp_packet *, void *);
93 static inline size_t dp_packet_l4_size(const struct dp_packet *);
94 static inline const void *dp_packet_get_tcp_payload(const struct dp_packet *);
95 static inline const void *dp_packet_get_udp_payload(const struct dp_packet *);
96 static inline const void *dp_packet_get_sctp_payload(const struct dp_packet *);
97 static inline const void *dp_packet_get_icmp_payload(const struct dp_packet *);
98 static inline const void *dp_packet_get_nd_payload(const struct dp_packet *);
100 void dp_packet_use(struct dp_packet *, void *, size_t);
101 void dp_packet_use_stub(struct dp_packet *, void *, size_t);
102 void dp_packet_use_const(struct dp_packet *, const void *, size_t);
104 void dp_packet_init_dpdk(struct dp_packet *, size_t allocated);
106 void dp_packet_init(struct dp_packet *, size_t);
107 void dp_packet_uninit(struct dp_packet *);
109 struct dp_packet *dp_packet_new(size_t);
110 struct dp_packet *dp_packet_new_with_headroom(size_t, size_t headroom);
111 struct dp_packet *dp_packet_clone(const struct dp_packet *);
112 struct dp_packet *dp_packet_clone_with_headroom(const struct dp_packet *,
114 struct dp_packet *dp_packet_clone_data(const void *, size_t);
115 struct dp_packet *dp_packet_clone_data_with_headroom(const void *, size_t,
117 static inline void dp_packet_delete(struct dp_packet *);
119 static inline void *dp_packet_at(const struct dp_packet *, size_t offset,
121 static inline void *dp_packet_at_assert(const struct dp_packet *,
122 size_t offset, size_t size);
123 static inline void *dp_packet_tail(const struct dp_packet *);
124 static inline void *dp_packet_end(const struct dp_packet *);
126 void *dp_packet_put_uninit(struct dp_packet *, size_t);
127 void *dp_packet_put_zeros(struct dp_packet *, size_t);
128 void *dp_packet_put(struct dp_packet *, const void *, size_t);
129 char *dp_packet_put_hex(struct dp_packet *, const char *s, size_t *n);
130 void dp_packet_reserve(struct dp_packet *, size_t);
131 void dp_packet_reserve_with_tailroom(struct dp_packet *, size_t headroom,
133 void *dp_packet_push_uninit(struct dp_packet *, size_t);
134 void *dp_packet_push_zeros(struct dp_packet *, size_t);
135 void *dp_packet_push(struct dp_packet *, const void *, size_t);
137 static inline size_t dp_packet_headroom(const struct dp_packet *);
138 static inline size_t dp_packet_tailroom(const struct dp_packet *);
139 void dp_packet_prealloc_headroom(struct dp_packet *, size_t);
140 void dp_packet_prealloc_tailroom(struct dp_packet *, size_t);
141 void dp_packet_shift(struct dp_packet *, int);
143 static inline void dp_packet_clear(struct dp_packet *);
144 static inline void *dp_packet_pull(struct dp_packet *, size_t);
145 static inline void *dp_packet_try_pull(struct dp_packet *, size_t);
147 void *dp_packet_steal_data(struct dp_packet *);
149 static inline bool dp_packet_equal(const struct dp_packet *,
150 const struct dp_packet *);
153 /* Frees memory that 'b' points to, as well as 'b' itself. */
155 dp_packet_delete(struct dp_packet *b)
158 if (b->source == DPBUF_DPDK) {
159 /* If this dp_packet was allocated by DPDK it must have been
160 * created as a dp_packet */
161 free_dpdk_buf((struct dp_packet*) b);
170 /* If 'b' contains at least 'offset + size' bytes of data, returns a pointer to
171 * byte 'offset'. Otherwise, returns a null pointer. */
173 dp_packet_at(const struct dp_packet *b, size_t offset, size_t size)
175 return offset + size <= dp_packet_size(b)
176 ? (char *) dp_packet_data(b) + offset
180 /* Returns a pointer to byte 'offset' in 'b', which must contain at least
181 * 'offset + size' bytes of data. */
183 dp_packet_at_assert(const struct dp_packet *b, size_t offset, size_t size)
185 ovs_assert(offset + size <= dp_packet_size(b));
186 return ((char *) dp_packet_data(b)) + offset;
189 /* Returns a pointer to byte following the last byte of data in use in 'b'. */
191 dp_packet_tail(const struct dp_packet *b)
193 return (char *) dp_packet_data(b) + dp_packet_size(b);
196 /* Returns a pointer to byte following the last byte allocated for use (but
197 * not necessarily in use) in 'b'. */
199 dp_packet_end(const struct dp_packet *b)
201 return (char *) dp_packet_base(b) + dp_packet_get_allocated(b);
204 /* Returns the number of bytes of headroom in 'b', that is, the number of bytes
205 * of unused space in dp_packet 'b' before the data that is in use. (Most
206 * commonly, the data in a dp_packet is at its beginning, and thus the
207 * dp_packet's headroom is 0.) */
209 dp_packet_headroom(const struct dp_packet *b)
211 return (char *) dp_packet_data(b) - (char *) dp_packet_base(b);
214 /* Returns the number of bytes that may be appended to the tail end of
215 * dp_packet 'b' before the dp_packet must be reallocated. */
217 dp_packet_tailroom(const struct dp_packet *b)
219 return (char *) dp_packet_end(b) - (char *) dp_packet_tail(b);
222 /* Clears any data from 'b'. */
224 dp_packet_clear(struct dp_packet *b)
226 dp_packet_set_data(b, dp_packet_base(b));
227 dp_packet_set_size(b, 0);
230 /* Removes 'size' bytes from the head end of 'b', which must contain at least
231 * 'size' bytes of data. Returns the first byte of data removed. */
233 dp_packet_pull(struct dp_packet *b, size_t size)
235 void *data = dp_packet_data(b);
236 ovs_assert(dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size);
237 dp_packet_set_data(b, (char *) dp_packet_data(b) + size);
238 dp_packet_set_size(b, dp_packet_size(b) - size);
242 /* If 'b' has at least 'size' bytes of data, removes that many bytes from the
243 * head end of 'b' and returns the first byte removed. Otherwise, returns a
244 * null pointer without modifying 'b'. */
246 dp_packet_try_pull(struct dp_packet *b, size_t size)
248 return dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size
249 ? dp_packet_pull(b, size) : NULL;
253 dp_packet_equal(const struct dp_packet *a, const struct dp_packet *b)
255 return dp_packet_size(a) == dp_packet_size(b) &&
256 !memcmp(dp_packet_data(a), dp_packet_data(b), dp_packet_size(a));
259 /* Get the start of the Ethernet frame. 'l3_ofs' marks the end of the l2
260 * headers, so return NULL if it is not set. */
262 dp_packet_l2(const struct dp_packet *b)
264 return (b->l3_ofs != UINT16_MAX) ? dp_packet_data(b) : NULL;
267 /* Resets all layer offsets. 'l3' offset must be set before 'l2' can be
270 dp_packet_reset_offsets(struct dp_packet *b)
273 b->l2_5_ofs = UINT16_MAX;
274 b->l3_ofs = UINT16_MAX;
275 b->l4_ofs = UINT16_MAX;
278 static inline uint8_t
279 dp_packet_l2_pad_size(const struct dp_packet *b)
281 return b->l2_pad_size;
285 dp_packet_set_l2_pad_size(struct dp_packet *b, uint8_t pad_size)
287 ovs_assert(pad_size <= dp_packet_size(b));
288 b->l2_pad_size = pad_size;
292 dp_packet_l2_5(const struct dp_packet *b)
294 return b->l2_5_ofs != UINT16_MAX
295 ? (char *) dp_packet_data(b) + b->l2_5_ofs
300 dp_packet_set_l2_5(struct dp_packet *b, void *l2_5)
303 ? (char *) l2_5 - (char *) dp_packet_data(b)
308 dp_packet_l3(const struct dp_packet *b)
310 return b->l3_ofs != UINT16_MAX
311 ? (char *) dp_packet_data(b) + b->l3_ofs
316 dp_packet_set_l3(struct dp_packet *b, void *l3)
318 b->l3_ofs = l3 ? (char *) l3 - (char *) dp_packet_data(b) : UINT16_MAX;
322 dp_packet_l4(const struct dp_packet *b)
324 return b->l4_ofs != UINT16_MAX
325 ? (char *) dp_packet_data(b) + b->l4_ofs
330 dp_packet_set_l4(struct dp_packet *b, void *l4)
332 b->l4_ofs = l4 ? (char *) l4 - (char *) dp_packet_data(b) : UINT16_MAX;
336 dp_packet_l4_size(const struct dp_packet *b)
338 return b->l4_ofs != UINT16_MAX
339 ? (const char *)dp_packet_tail(b) - (const char *)dp_packet_l4(b)
340 - dp_packet_l2_pad_size(b)
344 static inline const void *
345 dp_packet_get_tcp_payload(const struct dp_packet *b)
347 size_t l4_size = dp_packet_l4_size(b);
349 if (OVS_LIKELY(l4_size >= TCP_HEADER_LEN)) {
350 struct tcp_header *tcp = dp_packet_l4(b);
351 int tcp_len = TCP_OFFSET(tcp->tcp_ctl) * 4;
353 if (OVS_LIKELY(tcp_len >= TCP_HEADER_LEN && tcp_len <= l4_size)) {
354 return (const char *)tcp + tcp_len;
360 static inline const void *
361 dp_packet_get_udp_payload(const struct dp_packet *b)
363 return OVS_LIKELY(dp_packet_l4_size(b) >= UDP_HEADER_LEN)
364 ? (const char *)dp_packet_l4(b) + UDP_HEADER_LEN : NULL;
367 static inline const void *
368 dp_packet_get_sctp_payload(const struct dp_packet *b)
370 return OVS_LIKELY(dp_packet_l4_size(b) >= SCTP_HEADER_LEN)
371 ? (const char *)dp_packet_l4(b) + SCTP_HEADER_LEN : NULL;
374 static inline const void *
375 dp_packet_get_icmp_payload(const struct dp_packet *b)
377 return OVS_LIKELY(dp_packet_l4_size(b) >= ICMP_HEADER_LEN)
378 ? (const char *)dp_packet_l4(b) + ICMP_HEADER_LEN : NULL;
381 static inline const void *
382 dp_packet_get_nd_payload(const struct dp_packet *b)
384 return OVS_LIKELY(dp_packet_l4_size(b) >= ND_MSG_LEN)
385 ? (const char *)dp_packet_l4(b) + ND_MSG_LEN : NULL;
389 BUILD_ASSERT_DECL(offsetof(struct dp_packet, mbuf) == 0);
392 dp_packet_base(const struct dp_packet *b)
394 return b->mbuf.buf_addr;
398 dp_packet_set_base(struct dp_packet *b, void *d)
400 b->mbuf.buf_addr = d;
403 static inline uint32_t
404 dp_packet_size(const struct dp_packet *b)
406 return b->mbuf.pkt_len;
410 dp_packet_set_size(struct dp_packet *b, uint32_t v)
412 /* netdev-dpdk does not currently support segmentation; consequently, for
413 * all intents and purposes, 'data_len' (16 bit) and 'pkt_len' (32 bit) may
414 * be used interchangably.
416 * On the datapath, it is expected that the size of packets
417 * (and thus 'v') will always be <= UINT16_MAX; this means that there is no
418 * loss of accuracy in assigning 'v' to 'data_len'.
420 b->mbuf.data_len = (uint16_t)v; /* Current seg length. */
421 b->mbuf.pkt_len = v; /* Total length of all segments linked to
425 static inline uint16_t
426 __packet_data(const struct dp_packet *b)
428 return b->mbuf.data_off;
432 __packet_set_data(struct dp_packet *b, uint16_t v)
434 b->mbuf.data_off = v;
437 static inline uint16_t
438 dp_packet_get_allocated(const struct dp_packet *b)
440 return b->mbuf.buf_len;
444 dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
450 dp_packet_base(const struct dp_packet *b)
456 dp_packet_set_base(struct dp_packet *b, void *d)
461 static inline uint32_t
462 dp_packet_size(const struct dp_packet *b)
468 dp_packet_set_size(struct dp_packet *b, uint32_t v)
473 static inline uint16_t
474 __packet_data(const struct dp_packet *b)
480 __packet_set_data(struct dp_packet *b, uint16_t v)
485 static inline uint16_t
486 dp_packet_get_allocated(const struct dp_packet *b)
488 return b->allocated_;
492 dp_packet_set_allocated(struct dp_packet *b, uint16_t s)
499 dp_packet_reset_cutlen(struct dp_packet *b)
504 static inline uint32_t
505 dp_packet_set_cutlen(struct dp_packet *b, uint32_t max_len)
507 if (max_len < ETH_HEADER_LEN) {
508 max_len = ETH_HEADER_LEN;
511 if (max_len >= dp_packet_size(b)) {
514 b->cutlen = dp_packet_size(b) - max_len;
519 static inline uint32_t
520 dp_packet_get_cutlen(struct dp_packet *b)
522 /* Always in valid range if user uses dp_packet_set_cutlen. */
527 dp_packet_data(const struct dp_packet *b)
529 return __packet_data(b) != UINT16_MAX
530 ? (char *) dp_packet_base(b) + __packet_data(b) : NULL;
534 dp_packet_set_data(struct dp_packet *b, void *data)
537 __packet_set_data(b, (char *) data - (char *) dp_packet_base(b));
539 __packet_set_data(b, UINT16_MAX);
544 dp_packet_reset_packet(struct dp_packet *b, int off)
546 dp_packet_set_size(b, dp_packet_size(b) - off);
547 dp_packet_set_data(b, ((unsigned char *) dp_packet_data(b) + off));
548 dp_packet_reset_offsets(b);
551 /* Returns the RSS hash of the packet 'p'. Note that the returned value is
552 * correct only if 'dp_packet_rss_valid(p)' returns true */
553 static inline uint32_t
554 dp_packet_get_rss_hash(struct dp_packet *p)
557 return p->mbuf.hash.rss;
564 dp_packet_set_rss_hash(struct dp_packet *p, uint32_t hash)
567 p->mbuf.hash.rss = hash;
568 p->mbuf.ol_flags |= PKT_RX_RSS_HASH;
571 p->rss_hash_valid = true;
576 dp_packet_rss_valid(struct dp_packet *p)
579 return p->mbuf.ol_flags & PKT_RX_RSS_HASH;
581 return p->rss_hash_valid;
586 dp_packet_rss_invalidate(struct dp_packet *p)
589 p->mbuf.ol_flags &= ~PKT_RX_RSS_HASH;
591 p->rss_hash_valid = false;
595 enum { NETDEV_MAX_BURST = 32 }; /* Maximum number packets in a batch. */
597 struct dp_packet_batch {
599 bool trunc; /* true if the batch needs truncate. */
600 struct dp_packet *packets[NETDEV_MAX_BURST];
603 static inline void dp_packet_batch_init(struct dp_packet_batch *b)
610 dp_packet_batch_clone(struct dp_packet_batch *dst,
611 struct dp_packet_batch *src)
615 for (i = 0; i < src->count; i++) {
616 dst->packets[i] = dp_packet_clone(src->packets[i]);
618 dst->count = src->count;
619 dst->trunc = src->trunc;
623 packet_batch_init_packet(struct dp_packet_batch *b, struct dp_packet *p)
631 dp_packet_delete_batch(struct dp_packet_batch *batch, bool may_steal)
636 for (i = 0; i < batch->count; i++) {
637 dp_packet_delete(batch->packets[i]);
643 dp_packet_batch_apply_cutlen(struct dp_packet_batch *pktb)
650 for (i = 0; i < pktb->count; i++) {
651 uint32_t cutlen = dp_packet_get_cutlen(pktb->packets[i]);
653 dp_packet_set_size(pktb->packets[i],
654 dp_packet_size(pktb->packets[i]) - cutlen);
655 dp_packet_reset_cutlen(pktb->packets[i]);
661 dp_packet_batch_reset_cutlen(struct dp_packet_batch *pktb)
669 for (i = 0; i < pktb->count; i++) {
670 dp_packet_reset_cutlen(pktb->packets[i]);
678 #endif /* dp-packet.h */