+/* U64 indices for segmented flow classification. */
+const uint8_t flow_segment_u64s[4] = {
+ FLOW_SEGMENT_1_ENDS_AT / sizeof(uint64_t),
+ FLOW_SEGMENT_2_ENDS_AT / sizeof(uint64_t),
+ FLOW_SEGMENT_3_ENDS_AT / sizeof(uint64_t),
+ FLOW_U64S
+};
+
+/* Asserts that field 'f1' follows immediately after 'f0' in struct flow,
+ * without any intervening padding. */
+#define ASSERT_SEQUENTIAL(f0, f1) \
+ BUILD_ASSERT_DECL(offsetof(struct flow, f0) \
+ + MEMBER_SIZEOF(struct flow, f0) \
+ == offsetof(struct flow, f1))
+
+/* Asserts that fields 'f0' and 'f1' are in the same 32-bit aligned word within
+ * struct flow. */
+#define ASSERT_SAME_WORD(f0, f1) \
+ BUILD_ASSERT_DECL(offsetof(struct flow, f0) / 4 \
+ == offsetof(struct flow, f1) / 4)
+
+/* Asserts that 'f0' and 'f1' are both sequential and within the same 32-bit
+ * aligned word in struct flow. */
+#define ASSERT_SEQUENTIAL_SAME_WORD(f0, f1) \
+ ASSERT_SEQUENTIAL(f0, f1); \
+ ASSERT_SAME_WORD(f0, f1)
+
+/* miniflow_extract() assumes the following to be true to optimize the
+ * extraction process. */
+ASSERT_SEQUENTIAL_SAME_WORD(dl_type, vlan_tci);
+
+ASSERT_SEQUENTIAL_SAME_WORD(nw_frag, nw_tos);
+ASSERT_SEQUENTIAL_SAME_WORD(nw_tos, nw_ttl);
+ASSERT_SEQUENTIAL_SAME_WORD(nw_ttl, nw_proto);
+
+/* TCP flags in the middle of a BE64, zeroes in the other half. */
+BUILD_ASSERT_DECL(offsetof(struct flow, tcp_flags) % 8 == 4);
+
+#if WORDS_BIGENDIAN
+#define TCP_FLAGS_BE32(tcp_ctl) ((OVS_FORCE ovs_be32)TCP_FLAGS_BE16(tcp_ctl) \
+ << 16)
+#else
+#define TCP_FLAGS_BE32(tcp_ctl) ((OVS_FORCE ovs_be32)TCP_FLAGS_BE16(tcp_ctl))
+#endif
+
+ASSERT_SEQUENTIAL_SAME_WORD(tp_src, tp_dst);
+
+/* Removes 'size' bytes from the head end of '*datap', of size '*sizep', which
+ * must contain at least 'size' bytes of data. Returns the first byte of data
+ * removed. */
+static inline const void *
+data_pull(const void **datap, size_t *sizep, size_t size)
+{
+ const char *data = *datap;
+ *datap = data + size;
+ *sizep -= size;
+ return data;
+}
+
+/* If '*datap' has at least 'size' bytes of data, removes that many bytes from
+ * the head end of '*datap' and returns the first byte removed. Otherwise,
+ * returns a null pointer without modifying '*datap'. */
+static inline const void *
+data_try_pull(const void **datap, size_t *sizep, size_t size)
+{
+ return OVS_LIKELY(*sizep >= size) ? data_pull(datap, sizep, size) : NULL;
+}
+
+/* Context for pushing data to a miniflow. */
+struct mf_ctx {
+ struct flowmap map;
+ uint64_t *data;
+ uint64_t * const end;
+};
+
+/* miniflow_push_* macros allow filling in a miniflow data values in order.
+ * Assertions are needed only when the layout of the struct flow is modified.
+ * 'ofs' is a compile-time constant, which allows most of the code be optimized
+ * away. Some GCC versions gave warnings on ALWAYS_INLINE, so these are
+ * defined as macros. */
+
+#if (FLOW_WC_SEQ != 35)
+#define MINIFLOW_ASSERT(X) ovs_assert(X)
+BUILD_MESSAGE("FLOW_WC_SEQ changed: miniflow_extract() will have runtime "
+ "assertions enabled. Consider updating FLOW_WC_SEQ after "
+ "testing")
+#else
+#define MINIFLOW_ASSERT(X)
+#endif
+
+/* True if 'IDX' and higher bits are not set. */
+#define ASSERT_FLOWMAP_NOT_SET(FM, IDX) \
+{ \
+ MINIFLOW_ASSERT(!((FM)->bits[(IDX) / MAP_T_BITS] & \
+ (MAP_MAX << ((IDX) % MAP_T_BITS)))); \
+ for (size_t i = (IDX) / MAP_T_BITS + 1; i < FLOWMAP_UNITS; i++) { \
+ MINIFLOW_ASSERT(!(FM)->bits[i]); \
+ } \
+}
+
+#define miniflow_set_map(MF, OFS) \
+ { \
+ ASSERT_FLOWMAP_NOT_SET(&MF.map, (OFS)); \
+ flowmap_set(&MF.map, (OFS), 1); \
+}
+
+#define miniflow_assert_in_map(MF, OFS) \
+ MINIFLOW_ASSERT(flowmap_is_set(&MF.map, (OFS))); \
+ ASSERT_FLOWMAP_NOT_SET(&MF.map, (OFS) + 1)
+
+#define miniflow_push_uint64_(MF, OFS, VALUE) \
+{ \
+ MINIFLOW_ASSERT(MF.data < MF.end && (OFS) % 8 == 0); \
+ *MF.data++ = VALUE; \
+ miniflow_set_map(MF, OFS / 8); \
+}
+
+#define miniflow_push_be64_(MF, OFS, VALUE) \
+ miniflow_push_uint64_(MF, OFS, (OVS_FORCE uint64_t)(VALUE))
+
+#define miniflow_push_uint32_(MF, OFS, VALUE) \
+ { \
+ MINIFLOW_ASSERT(MF.data < MF.end); \
+ \
+ if ((OFS) % 8 == 0) { \
+ miniflow_set_map(MF, OFS / 8); \
+ *(uint32_t *)MF.data = VALUE; \
+ } else if ((OFS) % 8 == 4) { \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ *((uint32_t *)MF.data + 1) = VALUE; \
+ MF.data++; \
+ } \
+}
+
+#define miniflow_push_be32_(MF, OFS, VALUE) \
+ miniflow_push_uint32_(MF, OFS, (OVS_FORCE uint32_t)(VALUE))
+
+#define miniflow_push_uint16_(MF, OFS, VALUE) \
+{ \
+ MINIFLOW_ASSERT(MF.data < MF.end); \
+ \
+ if ((OFS) % 8 == 0) { \
+ miniflow_set_map(MF, OFS / 8); \
+ *(uint16_t *)MF.data = VALUE; \
+ } else if ((OFS) % 8 == 2) { \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ *((uint16_t *)MF.data + 1) = VALUE; \
+ } else if ((OFS) % 8 == 4) { \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ *((uint16_t *)MF.data + 2) = VALUE; \
+ } else if ((OFS) % 8 == 6) { \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ *((uint16_t *)MF.data + 3) = VALUE; \
+ MF.data++; \
+ } \
+}
+
+#define miniflow_push_uint8_(MF, OFS, VALUE) \
+{ \
+ MINIFLOW_ASSERT(MF.data < MF.end); \
+ \
+ if ((OFS) % 8 == 0) { \
+ miniflow_set_map(MF, OFS / 8); \
+ *(uint8_t *)MF.data = VALUE; \
+ } else if ((OFS) % 8 == 7) { \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ *((uint8_t *)MF.data + 7) = VALUE; \
+ MF.data++; \
+ } else { \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ *((uint8_t *)MF.data + ((OFS) % 8)) = VALUE; \
+ } \
+}
+
+#define miniflow_pad_to_64_(MF, OFS) \
+{ \
+ MINIFLOW_ASSERT((OFS) % 8 != 0); \
+ miniflow_assert_in_map(MF, OFS / 8); \
+ \
+ memset((uint8_t *)MF.data + (OFS) % 8, 0, 8 - (OFS) % 8); \
+ MF.data++; \
+}
+
+#define miniflow_pad_from_64_(MF, OFS) \
+{ \
+ MINIFLOW_ASSERT(MF.data < MF.end); \
+ \
+ MINIFLOW_ASSERT((OFS) % 8 != 0); \
+ miniflow_set_map(MF, OFS / 8); \
+ \
+ memset((uint8_t *)MF.data, 0, (OFS) % 8); \
+}
+
+#define miniflow_push_be16_(MF, OFS, VALUE) \
+ miniflow_push_uint16_(MF, OFS, (OVS_FORCE uint16_t)VALUE);
+
+#define miniflow_push_be8_(MF, OFS, VALUE) \
+ miniflow_push_uint8_(MF, OFS, (OVS_FORCE uint8_t)VALUE);
+
+#define miniflow_set_maps(MF, OFS, N_WORDS) \
+{ \
+ size_t ofs = (OFS); \
+ size_t n_words = (N_WORDS); \
+ \
+ MINIFLOW_ASSERT(n_words && MF.data + n_words <= MF.end); \
+ ASSERT_FLOWMAP_NOT_SET(&MF.map, ofs); \
+ flowmap_set(&MF.map, ofs, n_words); \
+}
+
+/* Data at 'valuep' may be unaligned. */
+#define miniflow_push_words_(MF, OFS, VALUEP, N_WORDS) \
+{ \
+ MINIFLOW_ASSERT((OFS) % 8 == 0); \
+ miniflow_set_maps(MF, (OFS) / 8, (N_WORDS)); \
+ memcpy(MF.data, (VALUEP), (N_WORDS) * sizeof *MF.data); \
+ MF.data += (N_WORDS); \
+}
+
+/* Push 32-bit words padded to 64-bits. */
+#define miniflow_push_words_32_(MF, OFS, VALUEP, N_WORDS) \
+{ \
+ miniflow_set_maps(MF, (OFS) / 8, DIV_ROUND_UP(N_WORDS, 2)); \
+ memcpy(MF.data, (VALUEP), (N_WORDS) * sizeof(uint32_t)); \
+ MF.data += DIV_ROUND_UP(N_WORDS, 2); \
+ if ((N_WORDS) & 1) { \
+ *((uint32_t *)MF.data - 1) = 0; \
+ } \
+}
+
+/* Data at 'valuep' may be unaligned. */
+/* MACs start 64-aligned, and must be followed by other data or padding. */
+#define miniflow_push_macs_(MF, OFS, VALUEP) \
+{ \
+ miniflow_set_maps(MF, (OFS) / 8, 2); \
+ memcpy(MF.data, (VALUEP), 2 * ETH_ADDR_LEN); \
+ MF.data += 1; /* First word only. */ \
+}
+
+#define miniflow_push_uint32(MF, FIELD, VALUE) \
+ miniflow_push_uint32_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_push_be32(MF, FIELD, VALUE) \
+ miniflow_push_be32_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_push_uint16(MF, FIELD, VALUE) \
+ miniflow_push_uint16_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_push_be16(MF, FIELD, VALUE) \
+ miniflow_push_be16_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_push_uint8(MF, FIELD, VALUE) \
+ miniflow_push_uint8_(MF, offsetof(struct flow, FIELD), VALUE)
+
+#define miniflow_pad_to_64(MF, FIELD) \
+ miniflow_pad_to_64_(MF, OFFSETOFEND(struct flow, FIELD))