ofproto/ofproto-dpif-ipfix.c

   1 /*
   2  * Copyright (c) 2012, 2013, 2014, 2015, 2016 Nicira, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <config.h>
  18 #include "ofproto-dpif-ipfix.h"
  19 #include <sys/time.h>
  20 #include "byte-order.h"
  21 #include "collectors.h"
  22 #include "flow.h"
  23 #include "hash.h"
  24 #include "hmap.h"
  25 #include "netdev.h"
  26 #include "openvswitch/list.h"
  27 #include "openvswitch/ofpbuf.h"
  28 #include "ofproto.h"
  29 #include "ofproto-dpif.h"
  30 #include "dp-packet.h"
  31 #include "packets.h"
  32 #include "poll-loop.h"
  33 #include "sset.h"
  34 #include "util.h"
  35 #include "timeval.h"
  36 #include "openvswitch/vlog.h"
  37
  38 VLOG_DEFINE_THIS_MODULE(ipfix);
  39
  40 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
  41 static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
  42
  43 /* Cf. IETF RFC 5101 Section 10.3.4. */
  44 #define IPFIX_DEFAULT_COLLECTOR_PORT 4739
  45
  46 /* Cf. IETF RFC 5881 Setion 8. */
  47 #define BFD_CONTROL_DEST_PORT        3784
  48 #define BFD_ECHO_DEST_PORT           3785
  49
  50 enum ipfix_sampled_packet_type {
  51     IPFIX_SAMPLED_PKT_UNKNOWN = 0x00,
  52     IPFIX_SAMPLED_PKT_IPV4_OK = 0x01,
  53     IPFIX_SAMPLED_PKT_IPV6_OK = 0x02,
  54     IPFIX_SAMPLED_PKT_IPV4_ERROR = 0x03,
  55     IPFIX_SAMPLED_PKT_IPV6_ERROR = 0x04,
  56     IPFIX_SAMPLED_PKT_OTHERS = 0x05
  57 };
  58
  59 /* The standard layer2SegmentId (ID 351) element is included in vDS to send
  60  * the VxLAN tunnel's VNI. It is 64-bit long, the most significant byte is
  61  * used to indicate the type of tunnel (0x01 = VxLAN, 0x02 = GRE) and the three
  62  * least significant bytes hold the value of the layer 2 overlay network
  63  * segment identifier: a 24-bit VxLAN tunnel's VNI or a 24-bit GRE tunnel's
  64  * TNI. This is not compatible with STT, as implemented in OVS, as
  65  * its tunnel IDs is 64-bit.
  66  *
  67  * Two new enterprise information elements are defined which are similar to
  68  * laryerSegmentId but support 64-bit IDs:
  69  *     tunnelType (ID 891) and tunnelKey (ID 892).
  70  *
  71  * The enum dpif_ipfix_tunnel_type is to declare the types supported in the
  72  * tunnelType element.
  73  * The number of ipfix tunnel types includes two reserverd types: 0x04 and 0x06.
  74  */
  75 enum dpif_ipfix_tunnel_type {
  76     DPIF_IPFIX_TUNNEL_UNKNOWN = 0x00,
  77     DPIF_IPFIX_TUNNEL_VXLAN = 0x01,
  78     DPIF_IPFIX_TUNNEL_GRE = 0x02,
  79     DPIF_IPFIX_TUNNEL_LISP = 0x03,
  80     DPIF_IPFIX_TUNNEL_STT = 0x04,
  81     DPIF_IPFIX_TUNNEL_IPSEC_GRE = 0x05,
  82     DPIF_IPFIX_TUNNEL_GENEVE = 0x07,
  83     NUM_DPIF_IPFIX_TUNNEL
  84 };
  85
  86 typedef struct ofputil_ipfix_stats ofproto_ipfix_stats;
  87
  88 struct dpif_ipfix_port {
  89     struct hmap_node hmap_node; /* In struct dpif_ipfix's "tunnel_ports" hmap. */
  90     struct ofport *ofport;      /* To retrieve port stats. */
  91     odp_port_t odp_port;
  92     enum dpif_ipfix_tunnel_type tunnel_type;
  93     uint8_t tunnel_key_length;
  94 };
  95
  96 struct dpif_ipfix_exporter {
  97     struct collectors *collectors;
  98     uint32_t seq_number;
  99     time_t last_template_set_time;
 100     struct hmap cache_flow_key_map;  /* ipfix_flow_cache_entry. */
 101     struct ovs_list cache_flow_start_timestamp_list;  /* ipfix_flow_cache_entry. */
 102     uint32_t cache_active_timeout;  /* In seconds. */
 103     uint32_t cache_max_flows;
 104     char *virtual_obs_id;
 105     uint8_t virtual_obs_len;
 106
 107     ofproto_ipfix_stats stats;
 108 };
 109
 110 struct dpif_ipfix_bridge_exporter {
 111     struct dpif_ipfix_exporter exporter;
 112     struct ofproto_ipfix_bridge_exporter_options *options;
 113     uint32_t probability;
 114 };
 115
 116 struct dpif_ipfix_flow_exporter {
 117     struct dpif_ipfix_exporter exporter;
 118     struct ofproto_ipfix_flow_exporter_options *options;
 119 };
 120
 121 struct dpif_ipfix_flow_exporter_map_node {
 122     struct hmap_node node;
 123     struct dpif_ipfix_flow_exporter exporter;
 124 };
 125
 126 struct dpif_ipfix {
 127     struct dpif_ipfix_bridge_exporter bridge_exporter;
 128     struct hmap flow_exporter_map;  /* dpif_ipfix_flow_exporter_map_node. */
 129     struct hmap tunnel_ports;       /* Contains "struct dpif_ipfix_port"s.
 130                                      * It makes tunnel port lookups faster in
 131                                      * sampling upcalls. */
 132     struct ovs_refcount ref_cnt;
 133 };
 134
 135 #define IPFIX_VERSION 0x000a
 136
 137 /* When using UDP, IPFIX Template Records must be re-sent regularly.
 138  * The standard default interval is 10 minutes (600 seconds).
 139  * Cf. IETF RFC 5101 Section 10.3.6. */
 140 #define IPFIX_TEMPLATE_INTERVAL 600
 141
 142 /* Cf. IETF RFC 5101 Section 3.1. */
 143 OVS_PACKED(
 144 struct ipfix_header {
 145     ovs_be16 version;  /* IPFIX_VERSION. */
 146     ovs_be16 length;  /* Length in bytes including this header. */
 147     ovs_be32 export_time;  /* Seconds since the epoch. */
 148     ovs_be32 seq_number;  /* Message sequence number. */
 149     ovs_be32 obs_domain_id;  /* Observation Domain ID. */
 150 });
 151 BUILD_ASSERT_DECL(sizeof(struct ipfix_header) == 16);
 152
 153 #define IPFIX_SET_ID_TEMPLATE 2
 154 #define IPFIX_SET_ID_OPTION_TEMPLATE 3
 155
 156 /* Cf. IETF RFC 5101 Section 3.3.2. */
 157 OVS_PACKED(
 158 struct ipfix_set_header {
 159     ovs_be16 set_id;  /* IPFIX_SET_ID_* or valid template ID for Data Sets. */
 160     ovs_be16 length;  /* Length of the set in bytes including header. */
 161 });
 162 BUILD_ASSERT_DECL(sizeof(struct ipfix_set_header) == 4);
 163
 164 /* Alternatives for templates at each layer.  A template is defined by
 165  * a combination of one value for each layer. */
 166 enum ipfix_proto_l2 {
 167     IPFIX_PROTO_L2_ETH = 0,  /* No VLAN. */
 168     IPFIX_PROTO_L2_VLAN,
 169     NUM_IPFIX_PROTO_L2
 170 };
 171 enum ipfix_proto_l3 {
 172     IPFIX_PROTO_L3_UNKNOWN = 0,
 173     IPFIX_PROTO_L3_IPV4,
 174     IPFIX_PROTO_L3_IPV6,
 175     NUM_IPFIX_PROTO_L3
 176 };
 177 enum ipfix_proto_l4 {
 178     IPFIX_PROTO_L4_UNKNOWN = 0,
 179     IPFIX_PROTO_L4_TCP_UDP_SCTP,
 180     IPFIX_PROTO_L4_ICMP,
 181     NUM_IPFIX_PROTO_L4
 182 };
 183 enum ipfix_proto_tunnel {
 184     IPFIX_PROTO_NOT_TUNNELED = 0,
 185     IPFIX_PROTO_TUNNELED,  /* Support gre, lisp and vxlan. */
 186     NUM_IPFIX_PROTO_TUNNEL
 187 };
 188
 189 /* Any Template ID > 255 is usable for Template Records. */
 190 #define IPFIX_TEMPLATE_ID_MIN 256
 191
 192 /* Cf. IETF RFC 5101 Section 3.4.1. */
 193 OVS_PACKED(
 194 struct ipfix_template_record_header {
 195     ovs_be16 template_id;
 196     ovs_be16 field_count;
 197 });
 198 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_record_header) == 4);
 199
 200 enum ipfix_entity_id {
 201 /* standard IPFIX elements */
 202 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ID_##ENUM = ID,
 203 #include "ofproto/ipfix-entities.def"
 204 /* non-standard IPFIX elements */
 205 #define IPFIX_SET_ENTERPRISE(v) (((v) | 0x8000))
 206 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 207     IPFIX_ENTITY_ID_##ENUM = IPFIX_SET_ENTERPRISE(ID),
 208 #include "ofproto/ipfix-enterprise-entities.def"
 209 };
 210
 211 enum ipfix_entity_size {
 212 /* standard IPFIX elements */
 213 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 214 #include "ofproto/ipfix-entities.def"
 215 /* non-standard IPFIX elements */
 216 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 217     IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 218 #include "ofproto/ipfix-enterprise-entities.def"
 219 };
 220
 221 enum ipfix_entity_enterprise {
 222 /* standard IPFIX elements */
 223 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ENTERPRISE_##ENUM = 0,
 224 #include "ofproto/ipfix-entities.def"
 225 /* non-standard IPFIX elements */
 226 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 227     IPFIX_ENTITY_ENTERPRISE_##ENUM = ENTERPRISE,
 228 #include "ofproto/ipfix-enterprise-entities.def"
 229 };
 230
 231 OVS_PACKED(
 232 struct ipfix_template_field_specifier {
 233     ovs_be16 element_id;  /* IPFIX_ENTITY_ID_*. */
 234     ovs_be16 field_length;  /* Length of the field's value, in bytes.
 235                              * For Variable-Length element, it should be 65535.
 236                              */
 237     ovs_be32 enterprise;  /* Enterprise number */
 238 });
 239 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 8);
 240
 241 /* Cf. IETF RFC 5102 Section 5.11.6. */
 242 enum ipfix_flow_direction {
 243     INGRESS_FLOW = 0x00,
 244     EGRESS_FLOW = 0x01
 245 };
 246
 247 /* Part of data record flow key for common metadata and Ethernet entities. */
 248 OVS_PACKED(
 249 struct ipfix_data_record_flow_key_common {
 250     ovs_be32 observation_point_id;  /* OBSERVATION_POINT_ID */
 251     uint8_t flow_direction;  /* FLOW_DIRECTION */
 252     struct eth_addr source_mac_address; /* SOURCE_MAC_ADDRESS */
 253     struct eth_addr destination_mac_address; /* DESTINATION_MAC_ADDRESS */
 254     ovs_be16 ethernet_type;  /* ETHERNET_TYPE */
 255     uint8_t ethernet_header_length;  /* ETHERNET_HEADER_LENGTH */
 256 });
 257 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 20);
 258
 259 /* Part of data record flow key for VLAN entities. */
 260 OVS_PACKED(
 261 struct ipfix_data_record_flow_key_vlan {
 262     ovs_be16 vlan_id;  /* VLAN_ID */
 263     ovs_be16 dot1q_vlan_id;  /* DOT1Q_VLAN_ID */
 264     uint8_t dot1q_priority;  /* DOT1Q_PRIORITY */
 265 });
 266 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_vlan) == 5);
 267
 268 /* Part of data record flow key for IP entities. */
 269 /* XXX: Replace IP_TTL with MINIMUM_TTL and MAXIMUM_TTL? */
 270 OVS_PACKED(
 271 struct ipfix_data_record_flow_key_ip {
 272     uint8_t ip_version;  /* IP_VERSION */
 273     uint8_t ip_ttl;  /* IP_TTL */
 274     uint8_t protocol_identifier;  /* PROTOCOL_IDENTIFIER */
 275     uint8_t ip_diff_serv_code_point;  /* IP_DIFF_SERV_CODE_POINT */
 276     uint8_t ip_precedence;  /* IP_PRECEDENCE */
 277     uint8_t ip_class_of_service;  /* IP_CLASS_OF_SERVICE */
 278 });
 279 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ip) == 6);
 280
 281 /* Part of data record flow key for IPv4 entities. */
 282 OVS_PACKED(
 283 struct ipfix_data_record_flow_key_ipv4 {
 284     ovs_be32 source_ipv4_address;  /* SOURCE_IPV4_ADDRESS */
 285     ovs_be32 destination_ipv4_address;  /* DESTINATION_IPV4_ADDRESS */
 286 });
 287 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ipv4) == 8);
 288
 289 /* Part of data record flow key for IPv6 entities. */
 290 OVS_PACKED(
 291 struct ipfix_data_record_flow_key_ipv6 {
 292     uint8_t source_ipv6_address[16];  /* SOURCE_IPV6_ADDRESS */
 293     uint8_t destination_ipv6_address[16];  /* DESTINATION_IPV6_ADDRESS */
 294     ovs_be32 flow_label_ipv6;  /* FLOW_LABEL_IPV6 */
 295 });
 296 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ipv6) == 36);
 297
 298 /* Part of data record flow key for TCP/UDP/SCTP entities. */
 299 OVS_PACKED(
 300 struct ipfix_data_record_flow_key_transport {
 301     ovs_be16 source_transport_port;  /* SOURCE_TRANSPORT_PORT */
 302     ovs_be16 destination_transport_port;  /* DESTINATION_TRANSPORT_PORT */
 303 });
 304 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_transport) == 4);
 305
 306 /* Part of data record flow key for ICMP entities. */
 307 OVS_PACKED(
 308 struct ipfix_data_record_flow_key_icmp {
 309     uint8_t icmp_type;  /* ICMP_TYPE_IPV4 / ICMP_TYPE_IPV6 */
 310     uint8_t icmp_code;  /* ICMP_CODE_IPV4 / ICMP_CODE_IPV6 */
 311 });
 312 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_icmp) == 2);
 313
 314 /* For the tunnel type that is on the top of IPSec, the protocol identifier
 315  * of the upper tunnel type is used.
 316  */
 317 static uint8_t tunnel_protocol[NUM_DPIF_IPFIX_TUNNEL] = {
 318     0,              /* reserved */
 319     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_VXLAN */
 320     IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_GRE */
 321     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_LISP*/
 322     IPPROTO_TCP,    /* DPIF_IPFIX_TUNNEL_STT*/
 323     IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_IPSEC_GRE */
 324     0          ,    /* reserved */
 325     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_GENEVE*/
 326 };
 327
 328 OVS_PACKED(
 329 struct ipfix_data_record_flow_key_tunnel {
 330     ovs_be32 tunnel_source_ipv4_address;  /* TUNNEL_SOURCE_IPV4_ADDRESS */
 331     ovs_be32 tunnel_destination_ipv4_address;  /* TUNNEL_DESTINATION_IPV4_ADDRESS */
 332     uint8_t tunnel_protocol_identifier;  /* TUNNEL_PROTOCOL_IDENTIFIER */
 333     ovs_be16 tunnel_source_transport_port;  /* TUNNEL_SOURCE_TRANSPORT_PORT */
 334     ovs_be16 tunnel_destination_transport_port;  /* TUNNEL_DESTINATION_TRANSPORT_PORT */
 335     uint8_t tunnel_type;  /* TUNNEL_TYPE */
 336     uint8_t tunnel_key_length;  /* length of TUNNEL_KEY */
 337     uint8_t tunnel_key[];  /* data of  TUNNEL_KEY */
 338 });
 339 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_tunnel) == 15);
 340
 341 /* Cf. IETF RFC 5102 Section 5.11.3. */
 342 enum ipfix_flow_end_reason {
 343     IDLE_TIMEOUT = 0x01,
 344     ACTIVE_TIMEOUT = 0x02,
 345     END_OF_FLOW_DETECTED = 0x03,
 346     FORCED_END = 0x04,
 347     LACK_OF_RESOURCES = 0x05
 348 };
 349
 350 /* Part of data record for common aggregated elements. */
 351 OVS_PACKED(
 352 struct ipfix_data_record_aggregated_common {
 353     ovs_be32 flow_start_delta_microseconds; /* FLOW_START_DELTA_MICROSECONDS */
 354     ovs_be32 flow_end_delta_microseconds; /* FLOW_END_DELTA_MICROSECONDS */
 355     ovs_be64 packet_delta_count;  /* PACKET_DELTA_COUNT */
 356     ovs_be64 layer2_octet_delta_count;  /* LAYER2_OCTET_DELTA_COUNT */
 357     uint8_t flow_end_reason;  /* FLOW_END_REASON */
 358 });
 359 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_common) == 25);
 360
 361 /* Part of data record for IP aggregated elements. */
 362 OVS_PACKED(
 363 struct ipfix_data_record_aggregated_ip {
 364     ovs_be64 octet_delta_count;  /* OCTET_DELTA_COUNT */
 365     ovs_be64 octet_delta_sum_of_squares;  /* OCTET_DELTA_SUM_OF_SQUARES */
 366     ovs_be64 minimum_ip_total_length;  /* MINIMUM_IP_TOTAL_LENGTH */
 367     ovs_be64 maximum_ip_total_length;  /* MAXIMUM_IP_TOTAL_LENGTH */
 368 });
 369 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
 370
 371 /*
 372  * Refer to RFC 7011, the length of Variable length element is 0~65535:
 373  * In most case, it should be less than 255 octets:
 374  *  0                   1                   2                   3
 375  *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 376  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 377  *  | Length (< 255)|          Information Element                  |
 378  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 379  *  |                      ... continuing as needed                 |
 380  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 381  *
 382  * When it is greater than or equeal to 255 octets:
 383  *  0                   1                   2                   3
 384  *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 385  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 386  *  |      255      |      Length (0 to 65535)      |       IE      |
 387  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 388  *  |                      ... continuing as needed                 |
 389  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 390  *
 391  *
 392  * Now, only the virtual_obs_id whose length < 255 is implemented.
 393  */
 394
 395 #define IPFIX_VIRTUAL_OBS_MAX_LEN 254
 396
 397 /*
 398  * support tunnel key for:
 399  * VxLAN: 24-bit VIN,
 400  * GRE: 32-bit key,
 401  * LISP: 24-bit instance ID
 402  * STT: 64-bit key
 403  */
 404 #define MAX_TUNNEL_KEY_LEN 8
 405
 406 #define MAX_FLOW_KEY_LEN                                        \
 407     (sizeof(struct ipfix_data_record_flow_key_common)           \
 408      + sizeof(struct ipfix_data_record_flow_key_vlan)           \
 409      + sizeof(struct ipfix_data_record_flow_key_ip)             \
 410      + MAX(sizeof(struct ipfix_data_record_flow_key_ipv4),      \
 411            sizeof(struct ipfix_data_record_flow_key_ipv6))      \
 412      + MAX(sizeof(struct ipfix_data_record_flow_key_icmp),      \
 413            sizeof(struct ipfix_data_record_flow_key_transport)) \
 414      + sizeof(struct ipfix_data_record_flow_key_tunnel)         \
 415      + MAX_TUNNEL_KEY_LEN)
 416
 417 #define MAX_DATA_RECORD_LEN                                 \
 418     (MAX_FLOW_KEY_LEN                                       \
 419      + sizeof(struct ipfix_data_record_aggregated_common)   \
 420      + sizeof(struct ipfix_data_record_aggregated_ip))
 421
 422 /* Max length of a data set.  To simplify the implementation, each
 423  * data record is sent in a separate data set, so each data set
 424  * contains at most one data record. */
 425 #define MAX_DATA_SET_LEN             \
 426     (sizeof(struct ipfix_set_header) \
 427      + MAX_DATA_RECORD_LEN)
 428
 429 /* Max length of an IPFIX message. Arbitrarily set to accommodate low
 430  * MTU. */
 431 #define MAX_MESSAGE_LEN 1024
 432
 433 /* Cache structures. */
 434
 435 /* Flow key. */
 436 struct ipfix_flow_key {
 437     uint32_t obs_domain_id;
 438     uint16_t template_id;
 439     size_t flow_key_msg_part_size;
 440     uint64_t flow_key_msg_part[DIV_ROUND_UP(MAX_FLOW_KEY_LEN, 8)];
 441 };
 442
 443 /* Flow cache entry. */
 444 struct ipfix_flow_cache_entry {
 445     struct hmap_node flow_key_map_node;
 446     struct ovs_list cache_flow_start_timestamp_list_node;
 447     struct ipfix_flow_key flow_key;
 448     /* Common aggregated elements. */
 449     uint64_t flow_start_timestamp_usec;
 450     uint64_t flow_end_timestamp_usec;
 451     uint64_t packet_delta_count;
 452     uint64_t layer2_octet_delta_count;
 453     uint64_t octet_delta_count;
 454     uint64_t octet_delta_sum_of_squares;  /* 0 if not IP. */
 455     uint16_t minimum_ip_total_length;  /* 0 if not IP. */
 456     uint16_t maximum_ip_total_length;  /* 0 if not IP. */
 457 };
 458
 459 static void dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *, bool,
 460                                     const uint64_t, const uint32_t);
 461
 462 static void get_export_time_now(uint64_t *, uint32_t *);
 463
 464 static void dpif_ipfix_cache_expire_now(struct dpif_ipfix_exporter *, bool);
 465
 466 static bool
 467 nullable_string_is_equal(const char *a, const char *b)
 468 {
 469     return a ? b && !strcmp(a, b) : !b;
 470 }
 471
 472 static bool
 473 ofproto_ipfix_bridge_exporter_options_equal(
 474     const struct ofproto_ipfix_bridge_exporter_options *a,
 475     const struct ofproto_ipfix_bridge_exporter_options *b)
 476 {
 477     return (a->obs_domain_id == b->obs_domain_id
 478             && a->obs_point_id == b->obs_point_id
 479             && a->sampling_rate == b->sampling_rate
 480             && a->cache_active_timeout == b->cache_active_timeout
 481             && a->cache_max_flows == b->cache_max_flows
 482             && a->enable_tunnel_sampling == b->enable_tunnel_sampling
 483             && a->enable_input_sampling == b->enable_input_sampling
 484             && a->enable_output_sampling == b->enable_output_sampling
 485             && sset_equals(&a->targets, &b->targets)
 486             && nullable_string_is_equal(a->virtual_obs_id, b->virtual_obs_id));
 487 }
 488
 489 static struct ofproto_ipfix_bridge_exporter_options *
 490 ofproto_ipfix_bridge_exporter_options_clone(
 491     const struct ofproto_ipfix_bridge_exporter_options *old)
 492 {
 493     struct ofproto_ipfix_bridge_exporter_options *new =
 494         xmemdup(old, sizeof *old);
 495     sset_clone(&new->targets, &old->targets);
 496     new->virtual_obs_id = nullable_xstrdup(old->virtual_obs_id);
 497     return new;
 498 }
 499
 500 static void
 501 ofproto_ipfix_bridge_exporter_options_destroy(
 502     struct ofproto_ipfix_bridge_exporter_options *options)
 503 {
 504     if (options) {
 505         sset_destroy(&options->targets);
 506         free(options->virtual_obs_id);
 507         free(options);
 508     }
 509 }
 510
 511 static bool
 512 ofproto_ipfix_flow_exporter_options_equal(
 513     const struct ofproto_ipfix_flow_exporter_options *a,
 514     const struct ofproto_ipfix_flow_exporter_options *b)
 515 {
 516     return (a->collector_set_id == b->collector_set_id
 517             && a->cache_active_timeout == b->cache_active_timeout
 518             && a->cache_max_flows == b->cache_max_flows
 519             && a->enable_tunnel_sampling == b->enable_tunnel_sampling
 520             && sset_equals(&a->targets, &b->targets)
 521             && nullable_string_is_equal(a->virtual_obs_id, b->virtual_obs_id));
 522 }
 523
 524 static struct ofproto_ipfix_flow_exporter_options *
 525 ofproto_ipfix_flow_exporter_options_clone(
 526     const struct ofproto_ipfix_flow_exporter_options *old)
 527 {
 528     struct ofproto_ipfix_flow_exporter_options *new =
 529         xmemdup(old, sizeof *old);
 530     sset_clone(&new->targets, &old->targets);
 531     new->virtual_obs_id = nullable_xstrdup(old->virtual_obs_id);
 532     return new;
 533 }
 534
 535 static void
 536 ofproto_ipfix_flow_exporter_options_destroy(
 537     struct ofproto_ipfix_flow_exporter_options *options)
 538 {
 539     if (options) {
 540         sset_destroy(&options->targets);
 541         free(options->virtual_obs_id);
 542         free(options);
 543     }
 544 }
 545
 546 static void
 547 dpif_ipfix_exporter_init(struct dpif_ipfix_exporter *exporter)
 548 {
 549     exporter->collectors = NULL;
 550     exporter->seq_number = 1;
 551     exporter->last_template_set_time = 0;
 552     hmap_init(&exporter->cache_flow_key_map);
 553     ovs_list_init(&exporter->cache_flow_start_timestamp_list);
 554     exporter->cache_active_timeout = 0;
 555     exporter->cache_max_flows = 0;
 556     exporter->virtual_obs_id = NULL;
 557     exporter->virtual_obs_len = 0;
 558 }
 559
 560 static void
 561 dpif_ipfix_exporter_clear(struct dpif_ipfix_exporter *exporter)
 562 {
 563     /* Flush the cache with flow end reason "forced end." */
 564     dpif_ipfix_cache_expire_now(exporter, true);
 565
 566     collectors_destroy(exporter->collectors);
 567     exporter->collectors = NULL;
 568     exporter->seq_number = 1;
 569     exporter->last_template_set_time = 0;
 570     exporter->cache_active_timeout = 0;
 571     exporter->cache_max_flows = 0;
 572     free(exporter->virtual_obs_id);
 573     exporter->virtual_obs_id = NULL;
 574     exporter->virtual_obs_len = 0;
 575 }
 576
 577 static void
 578 dpif_ipfix_exporter_destroy(struct dpif_ipfix_exporter *exporter)
 579 {
 580     dpif_ipfix_exporter_clear(exporter);
 581     hmap_destroy(&exporter->cache_flow_key_map);
 582 }
 583
 584 static bool
 585 dpif_ipfix_exporter_set_options(struct dpif_ipfix_exporter *exporter,
 586                                 const struct sset *targets,
 587                                 const uint32_t cache_active_timeout,
 588                                 const uint32_t cache_max_flows,
 589                                 const char *virtual_obs_id)
 590 {
 591     size_t virtual_obs_len;
 592     collectors_destroy(exporter->collectors);
 593     collectors_create(targets, IPFIX_DEFAULT_COLLECTOR_PORT,
 594                       &exporter->collectors);
 595     if (exporter->collectors == NULL) {
 596         VLOG_WARN_RL(&rl, "no collectors could be initialized, "
 597                      "IPFIX exporter disabled");
 598         dpif_ipfix_exporter_clear(exporter);
 599         return false;
 600     }
 601     exporter->cache_active_timeout = cache_active_timeout;
 602     exporter->cache_max_flows = cache_max_flows;
 603     virtual_obs_len = virtual_obs_id ? strlen(virtual_obs_id) : 0;
 604     if (virtual_obs_len > IPFIX_VIRTUAL_OBS_MAX_LEN) {
 605         VLOG_WARN_RL(&rl, "Virtual obsevation ID too long (%d bytes), "
 606                      "should not be longer than %d bytes.",
 607                      exporter->virtual_obs_len, IPFIX_VIRTUAL_OBS_MAX_LEN);
 608         dpif_ipfix_exporter_clear(exporter);
 609         return false;
 610     }
 611     exporter->virtual_obs_len = virtual_obs_len;
 612     exporter->virtual_obs_id = nullable_xstrdup(virtual_obs_id);
 613     return true;
 614 }
 615
 616 static struct dpif_ipfix_port *
 617 dpif_ipfix_find_port(const struct dpif_ipfix *di,
 618                      odp_port_t odp_port) OVS_REQUIRES(mutex)
 619 {
 620     struct dpif_ipfix_port *dip;
 621
 622     HMAP_FOR_EACH_IN_BUCKET (dip, hmap_node, hash_odp_port(odp_port),
 623                              &di->tunnel_ports) {
 624         if (dip->odp_port == odp_port) {
 625             return dip;
 626         }
 627     }
 628     return NULL;
 629 }
 630
 631 static void
 632 dpif_ipfix_del_port(struct dpif_ipfix *di,
 633                       struct dpif_ipfix_port *dip)
 634     OVS_REQUIRES(mutex)
 635 {
 636     hmap_remove(&di->tunnel_ports, &dip->hmap_node);
 637     free(dip);
 638 }
 639
 640 void
 641 dpif_ipfix_add_tunnel_port(struct dpif_ipfix *di, struct ofport *ofport,
 642                            odp_port_t odp_port) OVS_EXCLUDED(mutex)
 643 {
 644     struct dpif_ipfix_port *dip;
 645     const char *type;
 646
 647     ovs_mutex_lock(&mutex);
 648     dip = dpif_ipfix_find_port(di, odp_port);
 649     if (dip) {
 650         dpif_ipfix_del_port(di, dip);
 651     }
 652
 653     type = netdev_get_type(ofport->netdev);
 654     if (type == NULL) {
 655         goto out;
 656     }
 657
 658     /* Add to table of tunnel ports. */
 659     dip = xmalloc(sizeof *dip);
 660     dip->ofport = ofport;
 661     dip->odp_port = odp_port;
 662     if (strcmp(type, "gre") == 0) {
 663         /* 32-bit key gre */
 664         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
 665         dip->tunnel_key_length = 4;
 666     } else if (strcmp(type, "ipsec_gre") == 0) {
 667         /* 32-bit key ipsec_gre */
 668         dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
 669         dip->tunnel_key_length = 4;
 670     } else if (strcmp(type, "vxlan") == 0) {
 671         dip->tunnel_type = DPIF_IPFIX_TUNNEL_VXLAN;
 672         dip->tunnel_key_length = 3;
 673     } else if (strcmp(type, "lisp") == 0) {
 674         dip->tunnel_type = DPIF_IPFIX_TUNNEL_LISP;
 675         dip->tunnel_key_length = 3;
 676     } else if (strcmp(type, "geneve") == 0) {
 677         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GENEVE;
 678         dip->tunnel_key_length = 3;
 679     } else if (strcmp(type, "stt") == 0) {
 680         dip->tunnel_type = DPIF_IPFIX_TUNNEL_STT;
 681         dip->tunnel_key_length = 8;
 682     } else {
 683         free(dip);
 684         goto out;
 685     }
 686     hmap_insert(&di->tunnel_ports, &dip->hmap_node, hash_odp_port(odp_port));
 687
 688 out:
 689     ovs_mutex_unlock(&mutex);
 690 }
 691
 692 void
 693 dpif_ipfix_del_tunnel_port(struct dpif_ipfix *di, odp_port_t odp_port)
 694     OVS_EXCLUDED(mutex)
 695 {
 696     struct dpif_ipfix_port *dip;
 697     ovs_mutex_lock(&mutex);
 698     dip = dpif_ipfix_find_port(di, odp_port);
 699     if (dip) {
 700         dpif_ipfix_del_port(di, dip);
 701     }
 702     ovs_mutex_unlock(&mutex);
 703 }
 704
 705 bool
 706 dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *di, odp_port_t odp_port)
 707     OVS_EXCLUDED(mutex)
 708 {
 709     struct dpif_ipfix_port *dip;
 710     ovs_mutex_lock(&mutex);
 711     dip = dpif_ipfix_find_port(di, odp_port);
 712     ovs_mutex_unlock(&mutex);
 713     return dip != NULL;
 714 }
 715
 716 static void
 717 dpif_ipfix_bridge_exporter_init(struct dpif_ipfix_bridge_exporter *exporter)
 718 {
 719     dpif_ipfix_exporter_init(&exporter->exporter);
 720     exporter->options = NULL;
 721     exporter->probability = 0;
 722 }
 723
 724 static void
 725 dpif_ipfix_bridge_exporter_clear(struct dpif_ipfix_bridge_exporter *exporter)
 726 {
 727     dpif_ipfix_exporter_clear(&exporter->exporter);
 728     ofproto_ipfix_bridge_exporter_options_destroy(exporter->options);
 729     exporter->options = NULL;
 730     exporter->probability = 0;
 731 }
 732
 733 static void
 734 dpif_ipfix_bridge_exporter_destroy(struct dpif_ipfix_bridge_exporter *exporter)
 735 {
 736     dpif_ipfix_bridge_exporter_clear(exporter);
 737     dpif_ipfix_exporter_destroy(&exporter->exporter);
 738 }
 739
 740 static void
 741 dpif_ipfix_bridge_exporter_set_options(
 742     struct dpif_ipfix_bridge_exporter *exporter,
 743     const struct ofproto_ipfix_bridge_exporter_options *options)
 744 {
 745     bool options_changed;
 746
 747     if (!options || sset_is_empty(&options->targets)) {
 748         /* No point in doing any work if there are no targets. */
 749         dpif_ipfix_bridge_exporter_clear(exporter);
 750         return;
 751     }
 752
 753     options_changed = (
 754         !exporter->options
 755         || !ofproto_ipfix_bridge_exporter_options_equal(
 756             options, exporter->options));
 757
 758     /* Configure collectors if options have changed or if we're
 759      * shortchanged in collectors (which indicates that opening one or
 760      * more of the configured collectors failed, so that we should
 761      * retry). */
 762     if (options_changed
 763         || collectors_count(exporter->exporter.collectors)
 764             < sset_count(&options->targets)) {
 765         if (!dpif_ipfix_exporter_set_options(
 766                 &exporter->exporter, &options->targets,
 767                 options->cache_active_timeout, options->cache_max_flows,
 768                 options->virtual_obs_id)) {
 769             return;
 770         }
 771     }
 772
 773     /* Avoid reconfiguring if options didn't change. */
 774     if (!options_changed) {
 775         return;
 776     }
 777
 778     ofproto_ipfix_bridge_exporter_options_destroy(exporter->options);
 779     exporter->options = ofproto_ipfix_bridge_exporter_options_clone(options);
 780     exporter->probability =
 781         MAX(1, UINT32_MAX / exporter->options->sampling_rate);
 782
 783     /* Run over the cache as some entries might have expired after
 784      * changing the timeouts. */
 785     dpif_ipfix_cache_expire_now(&exporter->exporter, false);
 786 }
 787
 788 static struct dpif_ipfix_flow_exporter_map_node*
 789 dpif_ipfix_find_flow_exporter_map_node(
 790     const struct dpif_ipfix *di, const uint32_t collector_set_id)
 791     OVS_REQUIRES(mutex)
 792 {
 793     struct dpif_ipfix_flow_exporter_map_node *exporter_node;
 794
 795     HMAP_FOR_EACH_WITH_HASH (exporter_node, node,
 796                              hash_int(collector_set_id, 0),
 797                              &di->flow_exporter_map) {
 798         if (exporter_node->exporter.options->collector_set_id
 799             == collector_set_id) {
 800             return exporter_node;
 801         }
 802     }
 803
 804     return NULL;
 805 }
 806
 807 static void
 808 dpif_ipfix_flow_exporter_init(struct dpif_ipfix_flow_exporter *exporter)
 809 {
 810     dpif_ipfix_exporter_init(&exporter->exporter);
 811     exporter->options = NULL;
 812 }
 813
 814 static void
 815 dpif_ipfix_flow_exporter_clear(struct dpif_ipfix_flow_exporter *exporter)
 816 {
 817     dpif_ipfix_exporter_clear(&exporter->exporter);
 818     ofproto_ipfix_flow_exporter_options_destroy(exporter->options);
 819     exporter->options = NULL;
 820 }
 821
 822 static void
 823 dpif_ipfix_flow_exporter_destroy(struct dpif_ipfix_flow_exporter *exporter)
 824 {
 825     dpif_ipfix_flow_exporter_clear(exporter);
 826     dpif_ipfix_exporter_destroy(&exporter->exporter);
 827 }
 828
 829 static bool
 830 dpif_ipfix_flow_exporter_set_options(
 831     struct dpif_ipfix_flow_exporter *exporter,
 832     const struct ofproto_ipfix_flow_exporter_options *options)
 833 {
 834     bool options_changed;
 835
 836     if (sset_is_empty(&options->targets)) {
 837         /* No point in doing any work if there are no targets. */
 838         dpif_ipfix_flow_exporter_clear(exporter);
 839         return true;
 840     }
 841
 842     options_changed = (
 843         !exporter->options
 844         || !ofproto_ipfix_flow_exporter_options_equal(
 845             options, exporter->options));
 846
 847     /* Configure collectors if options have changed or if we're
 848      * shortchanged in collectors (which indicates that opening one or
 849      * more of the configured collectors failed, so that we should
 850      * retry). */
 851     if (options_changed
 852         || collectors_count(exporter->exporter.collectors)
 853             < sset_count(&options->targets)) {
 854         if (!dpif_ipfix_exporter_set_options(
 855                 &exporter->exporter, &options->targets,
 856                 options->cache_active_timeout, options->cache_max_flows,
 857                 options->virtual_obs_id)) {
 858             return false;
 859         }
 860     }
 861
 862     /* Avoid reconfiguring if options didn't change. */
 863     if (!options_changed) {
 864         return true;
 865     }
 866
 867     ofproto_ipfix_flow_exporter_options_destroy(exporter->options);
 868     exporter->options = ofproto_ipfix_flow_exporter_options_clone(options);
 869
 870     /* Run over the cache as some entries might have expired after
 871      * changing the timeouts. */
 872     dpif_ipfix_cache_expire_now(&exporter->exporter, false);
 873
 874     return true;
 875 }
 876
 877 void
 878 dpif_ipfix_set_options(
 879     struct dpif_ipfix *di,
 880     const struct ofproto_ipfix_bridge_exporter_options *bridge_exporter_options,
 881     const struct ofproto_ipfix_flow_exporter_options *flow_exporters_options,
 882     size_t n_flow_exporters_options) OVS_EXCLUDED(mutex)
 883 {
 884     int i;
 885     struct ofproto_ipfix_flow_exporter_options *options;
 886     struct dpif_ipfix_flow_exporter_map_node *node, *next;
 887     size_t n_broken_flow_exporters_options = 0;
 888
 889     ovs_mutex_lock(&mutex);
 890     dpif_ipfix_bridge_exporter_set_options(&di->bridge_exporter,
 891                                            bridge_exporter_options);
 892
 893     /* Add new flow exporters and update current flow exporters. */
 894     options = (struct ofproto_ipfix_flow_exporter_options *)
 895         flow_exporters_options;
 896     for (i = 0; i < n_flow_exporters_options; i++) {
 897         node = dpif_ipfix_find_flow_exporter_map_node(
 898             di, options->collector_set_id);
 899         if (!node) {
 900             node = xzalloc(sizeof *node);
 901             dpif_ipfix_flow_exporter_init(&node->exporter);
 902             hmap_insert(&di->flow_exporter_map, &node->node,
 903                         hash_int(options->collector_set_id, 0));
 904         }
 905         if (!dpif_ipfix_flow_exporter_set_options(&node->exporter, options)) {
 906             n_broken_flow_exporters_options++;
 907         }
 908         options++;
 909     }
 910
 911     ovs_assert(hmap_count(&di->flow_exporter_map) >=
 912                (n_flow_exporters_options - n_broken_flow_exporters_options));
 913
 914     /* Remove dropped flow exporters, if any needs to be removed. */
 915     if (hmap_count(&di->flow_exporter_map) > n_flow_exporters_options) {
 916         HMAP_FOR_EACH_SAFE (node, next, node, &di->flow_exporter_map) {
 917             /* This is slow but doesn't take any extra memory, and
 918              * this table is not supposed to contain many rows anyway. */
 919             options = (struct ofproto_ipfix_flow_exporter_options *)
 920                 flow_exporters_options;
 921             for (i = 0; i < n_flow_exporters_options; i++) {
 922               if (node->exporter.options->collector_set_id
 923                   == options->collector_set_id) {
 924                   break;
 925               }
 926               options++;
 927             }
 928             if (i == n_flow_exporters_options) {  // Not found.
 929                 hmap_remove(&di->flow_exporter_map, &node->node);
 930                 dpif_ipfix_flow_exporter_destroy(&node->exporter);
 931                 free(node);
 932             }
 933         }
 934     }
 935
 936     ovs_assert(hmap_count(&di->flow_exporter_map) ==
 937                (n_flow_exporters_options - n_broken_flow_exporters_options));
 938     ovs_mutex_unlock(&mutex);
 939 }
 940
 941 struct dpif_ipfix *
 942 dpif_ipfix_create(void)
 943 {
 944     struct dpif_ipfix *di;
 945     di = xzalloc(sizeof *di);
 946     dpif_ipfix_bridge_exporter_init(&di->bridge_exporter);
 947     hmap_init(&di->flow_exporter_map);
 948     hmap_init(&di->tunnel_ports);
 949     ovs_refcount_init(&di->ref_cnt);
 950     return di;
 951 }
 952
 953 struct dpif_ipfix *
 954 dpif_ipfix_ref(const struct dpif_ipfix *di_)
 955 {
 956     struct dpif_ipfix *di = CONST_CAST(struct dpif_ipfix *, di_);
 957     if (di) {
 958         ovs_refcount_ref(&di->ref_cnt);
 959     }
 960     return di;
 961 }
 962
 963 uint32_t
 964 dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *di)
 965     OVS_EXCLUDED(mutex)
 966 {
 967     uint32_t ret;
 968     ovs_mutex_lock(&mutex);
 969     ret = di->bridge_exporter.probability;
 970     ovs_mutex_unlock(&mutex);
 971     return ret;
 972 }
 973
 974 bool
 975 dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *di)
 976     OVS_EXCLUDED(mutex)
 977 {
 978     bool ret = false;
 979     ovs_mutex_lock(&mutex);
 980     if (di->bridge_exporter.options) {
 981         ret = di->bridge_exporter.options->enable_input_sampling;
 982     }
 983     ovs_mutex_unlock(&mutex);
 984     return ret;
 985 }
 986
 987 bool
 988 dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *di)
 989     OVS_EXCLUDED(mutex)
 990 {
 991     bool ret = false;
 992     ovs_mutex_lock(&mutex);
 993     if (di->bridge_exporter.options) {
 994         ret = di->bridge_exporter.options->enable_output_sampling;
 995     }
 996     ovs_mutex_unlock(&mutex);
 997     return ret;
 998 }
 999
1000 bool
1001 dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *di)
1002     OVS_EXCLUDED(mutex)
1003 {
1004     bool ret = false;
1005     ovs_mutex_lock(&mutex);
1006     if (di->bridge_exporter.options) {
1007         ret = di->bridge_exporter.options->enable_tunnel_sampling;
1008     }
1009     ovs_mutex_unlock(&mutex);
1010     return ret;
1011 }
1012
1013 bool
1014 dpif_ipfix_get_flow_exporter_tunnel_sampling(const struct dpif_ipfix *di,
1015                                              const uint32_t collector_set_id)
1016     OVS_EXCLUDED(mutex)
1017 {
1018     ovs_mutex_lock(&mutex);
1019     struct dpif_ipfix_flow_exporter_map_node *node
1020         = dpif_ipfix_find_flow_exporter_map_node(di, collector_set_id);
1021     bool ret = (node
1022                 && node->exporter.options
1023                 && node->exporter.options->enable_tunnel_sampling);
1024     ovs_mutex_unlock(&mutex);
1025
1026     return ret;
1027 }
1028
1029 static void
1030 dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
1031 {
1032     struct dpif_ipfix_flow_exporter_map_node *exp_node;
1033     struct dpif_ipfix_port *dip, *next;
1034
1035     dpif_ipfix_bridge_exporter_clear(&di->bridge_exporter);
1036
1037     HMAP_FOR_EACH_POP (exp_node, node, &di->flow_exporter_map) {
1038         dpif_ipfix_flow_exporter_destroy(&exp_node->exporter);
1039         free(exp_node);
1040     }
1041
1042     HMAP_FOR_EACH_SAFE (dip, next, hmap_node, &di->tunnel_ports) {
1043         dpif_ipfix_del_port(di, dip);
1044     }
1045 }
1046
1047 void
1048 dpif_ipfix_unref(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
1049 {
1050     if (di && ovs_refcount_unref_relaxed(&di->ref_cnt) == 1) {
1051         ovs_mutex_lock(&mutex);
1052         dpif_ipfix_clear(di);
1053         dpif_ipfix_bridge_exporter_destroy(&di->bridge_exporter);
1054         hmap_destroy(&di->flow_exporter_map);
1055         hmap_destroy(&di->tunnel_ports);
1056         free(di);
1057         ovs_mutex_unlock(&mutex);
1058     }
1059 }
1060
1061 static void
1062 ipfix_init_header(uint32_t export_time_sec, uint32_t seq_number,
1063                   uint32_t obs_domain_id, struct dp_packet *msg)
1064 {
1065     struct ipfix_header *hdr;
1066
1067     hdr = dp_packet_put_zeros(msg, sizeof *hdr);
1068     hdr->version = htons(IPFIX_VERSION);
1069     hdr->length = htons(sizeof *hdr);  /* Updated in ipfix_send_msg. */
1070     hdr->export_time = htonl(export_time_sec);
1071     hdr->seq_number = htonl(seq_number);
1072     hdr->obs_domain_id = htonl(obs_domain_id);
1073 }
1074
1075 static size_t
1076 ipfix_send_msg(const struct collectors *collectors, struct dp_packet *msg)
1077 {
1078     struct ipfix_header *hdr;
1079     size_t tx_errors;
1080
1081     /* Adjust the length in the header. */
1082     hdr = dp_packet_data(msg);
1083     hdr->length = htons(dp_packet_size(msg));
1084
1085     tx_errors = collectors_send(collectors,
1086                                 dp_packet_data(msg), dp_packet_size(msg));
1087     dp_packet_set_size(msg, 0);
1088
1089     return tx_errors;
1090 }
1091
1092 static uint16_t
1093 ipfix_get_template_id(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
1094                       enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel)
1095 {
1096     uint16_t template_id;
1097     template_id = l2;
1098     template_id = template_id * NUM_IPFIX_PROTO_L3 + l3;
1099     template_id = template_id * NUM_IPFIX_PROTO_L4 + l4;
1100     template_id = template_id * NUM_IPFIX_PROTO_TUNNEL + tunnel;
1101     return IPFIX_TEMPLATE_ID_MIN + template_id;
1102 }
1103
1104 static void
1105 ipfix_define_template_entity(enum ipfix_entity_id id,
1106                              enum ipfix_entity_size size,
1107                              enum ipfix_entity_enterprise enterprise,
1108                              struct dp_packet *msg)
1109 {
1110     struct ipfix_template_field_specifier *field;
1111     size_t field_size;
1112
1113     if (enterprise) {
1114         field_size = sizeof *field;
1115     } else {
1116         /* No enterprise number */
1117         field_size = sizeof *field - sizeof(ovs_be32);
1118     }
1119     field = dp_packet_put_zeros(msg, field_size);
1120     field->element_id = htons(id);
1121     if (size) {
1122         field->field_length = htons(size);
1123     } else {
1124         /* RFC 5101, Section 7. Variable-Length Information Element */
1125         field->field_length = OVS_BE16_MAX;
1126     }
1127     if (enterprise) {
1128         field->enterprise = htonl(enterprise);
1129     }
1130
1131 }
1132
1133 static uint16_t
1134 ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
1135                              enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel,
1136                              bool virtual_obs_id_set,
1137                              struct dp_packet *msg)
1138 {
1139     uint16_t count = 0;
1140
1141 #define DEF(ID) \
1142     { \
1143         ipfix_define_template_entity(IPFIX_ENTITY_ID_##ID, \
1144                                      IPFIX_ENTITY_SIZE_##ID, \
1145                                      IPFIX_ENTITY_ENTERPRISE_##ID, msg); \
1146         count++; \
1147     }
1148
1149     /* 1. Flow key. */
1150
1151     DEF(OBSERVATION_POINT_ID);
1152     DEF(FLOW_DIRECTION);
1153
1154     /* Common Ethernet entities. */
1155     DEF(SOURCE_MAC_ADDRESS);
1156     DEF(DESTINATION_MAC_ADDRESS);
1157     DEF(ETHERNET_TYPE);
1158     DEF(ETHERNET_HEADER_LENGTH);
1159
1160     if (l2 == IPFIX_PROTO_L2_VLAN) {
1161         DEF(VLAN_ID);
1162         DEF(DOT1Q_VLAN_ID);
1163         DEF(DOT1Q_PRIORITY);
1164     }
1165
1166     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1167         DEF(IP_VERSION);
1168         DEF(IP_TTL);
1169         DEF(PROTOCOL_IDENTIFIER);
1170         DEF(IP_DIFF_SERV_CODE_POINT);
1171         DEF(IP_PRECEDENCE);
1172         DEF(IP_CLASS_OF_SERVICE);
1173
1174         if (l3 == IPFIX_PROTO_L3_IPV4) {
1175             DEF(SOURCE_IPV4_ADDRESS);
1176             DEF(DESTINATION_IPV4_ADDRESS);
1177             if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1178                 DEF(SOURCE_TRANSPORT_PORT);
1179                 DEF(DESTINATION_TRANSPORT_PORT);
1180             } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1181                 DEF(ICMP_TYPE_IPV4);
1182                 DEF(ICMP_CODE_IPV4);
1183             }
1184         } else {  /* l3 == IPFIX_PROTO_L3_IPV6 */
1185             DEF(SOURCE_IPV6_ADDRESS);
1186             DEF(DESTINATION_IPV6_ADDRESS);
1187             DEF(FLOW_LABEL_IPV6);
1188             if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1189                 DEF(SOURCE_TRANSPORT_PORT);
1190                 DEF(DESTINATION_TRANSPORT_PORT);
1191             } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1192                 DEF(ICMP_TYPE_IPV6);
1193                 DEF(ICMP_CODE_IPV6);
1194             }
1195         }
1196     }
1197
1198     if (tunnel != IPFIX_PROTO_NOT_TUNNELED) {
1199         DEF(TUNNEL_SOURCE_IPV4_ADDRESS);
1200         DEF(TUNNEL_DESTINATION_IPV4_ADDRESS);
1201         DEF(TUNNEL_PROTOCOL_IDENTIFIER);
1202         DEF(TUNNEL_SOURCE_TRANSPORT_PORT);
1203         DEF(TUNNEL_DESTINATION_TRANSPORT_PORT);
1204         DEF(TUNNEL_TYPE);
1205         DEF(TUNNEL_KEY);
1206     }
1207
1208     /* 2. Virtual observation ID, which is not a part of flow key. */
1209     if (virtual_obs_id_set) {
1210         DEF(VIRTUAL_OBS_ID);
1211     }
1212
1213     /* 3. Flow aggregated data. */
1214
1215     DEF(FLOW_START_DELTA_MICROSECONDS);
1216     DEF(FLOW_END_DELTA_MICROSECONDS);
1217     DEF(PACKET_DELTA_COUNT);
1218     DEF(LAYER2_OCTET_DELTA_COUNT);
1219     DEF(FLOW_END_REASON);
1220
1221     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1222         DEF(OCTET_DELTA_COUNT);
1223         DEF(OCTET_DELTA_SUM_OF_SQUARES);
1224         DEF(MINIMUM_IP_TOTAL_LENGTH);
1225         DEF(MAXIMUM_IP_TOTAL_LENGTH);
1226     }
1227 #undef DEF
1228
1229     return count;
1230 }
1231
1232 static void
1233 ipfix_init_template_msg(void *msg_stub, uint32_t export_time_sec,
1234                         uint32_t seq_number, uint32_t obs_domain_id,
1235                         struct dp_packet *msg, size_t *set_hdr_offset)
1236 {
1237     struct ipfix_set_header *set_hdr;
1238
1239     dp_packet_use_stub(msg, msg_stub, sizeof msg_stub);
1240
1241     ipfix_init_header(export_time_sec, seq_number, obs_domain_id, msg);
1242     *set_hdr_offset = dp_packet_size(msg);
1243
1244     /* Add a Template Set. */
1245     set_hdr = dp_packet_put_zeros(msg, sizeof *set_hdr);
1246     set_hdr->set_id = htons(IPFIX_SET_ID_TEMPLATE);
1247 }
1248
1249 static size_t
1250 ipfix_send_template_msg(const struct collectors *collectors,
1251                         struct dp_packet *msg, size_t set_hdr_offset)
1252 {
1253     struct ipfix_set_header *set_hdr;
1254     size_t tx_errors;
1255
1256     /* Send template message. */
1257     set_hdr = (struct ipfix_set_header*)
1258               ((uint8_t*)dp_packet_data(msg) + set_hdr_offset);
1259     set_hdr->length = htons(dp_packet_size(msg) - set_hdr_offset);
1260
1261     tx_errors = ipfix_send_msg(collectors, msg);
1262
1263     dp_packet_uninit(msg);
1264
1265     return tx_errors;
1266 }
1267
1268 static void
1269 ipfix_send_template_msgs(struct dpif_ipfix_exporter *exporter,
1270                          uint32_t export_time_sec, uint32_t obs_domain_id)
1271 {
1272     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
1273     struct dp_packet msg;
1274     size_t set_hdr_offset, tmpl_hdr_offset, error_pkts;
1275     struct ipfix_template_record_header *tmpl_hdr;
1276     uint16_t field_count;
1277     size_t tx_packets = 0;
1278     size_t tx_errors = 0;
1279     enum ipfix_proto_l2 l2;
1280     enum ipfix_proto_l3 l3;
1281     enum ipfix_proto_l4 l4;
1282     enum ipfix_proto_tunnel tunnel;
1283
1284     ipfix_init_template_msg(msg_stub, export_time_sec, exporter->seq_number,
1285                             obs_domain_id, &msg, &set_hdr_offset);
1286     /* Define one template for each possible combination of
1287      * protocols. */
1288     for (l2 = 0; l2 < NUM_IPFIX_PROTO_L2; l2++) {
1289         for (l3 = 0; l3 < NUM_IPFIX_PROTO_L3; l3++) {
1290             for (l4 = 0; l4 < NUM_IPFIX_PROTO_L4; l4++) {
1291                 if (l3 == IPFIX_PROTO_L3_UNKNOWN &&
1292                     l4 != IPFIX_PROTO_L4_UNKNOWN) {
1293                     continue;
1294                 }
1295                 for (tunnel = 0; tunnel < NUM_IPFIX_PROTO_TUNNEL; tunnel++) {
1296                     /* When the size of the template packet reaches
1297                      * MAX_MESSAGE_LEN(1024), send it out.
1298                      * And then reinitialize the msg to construct a new
1299                      * packet for the following templates.
1300                      */
1301                     if (dp_packet_size(&msg) >= MAX_MESSAGE_LEN) {
1302                         /* Send template message. */
1303                         error_pkts = ipfix_send_template_msg(exporter->collectors,
1304                                                              &msg, set_hdr_offset);
1305                         tx_errors += error_pkts;
1306                         tx_packets += collectors_count(exporter->collectors) - error_pkts;
1307
1308                         /* Reinitialize the template msg. */
1309                         ipfix_init_template_msg(msg_stub, export_time_sec,
1310                                                 exporter->seq_number,
1311                                                 obs_domain_id, &msg,
1312                                                 &set_hdr_offset);
1313                     }
1314
1315                     tmpl_hdr_offset = dp_packet_size(&msg);
1316                     tmpl_hdr = dp_packet_put_zeros(&msg, sizeof *tmpl_hdr);
1317                     tmpl_hdr->template_id = htons(
1318                         ipfix_get_template_id(l2, l3, l4, tunnel));
1319                     field_count = ipfix_define_template_fields(
1320                         l2, l3, l4, tunnel, exporter->virtual_obs_id != NULL,
1321                         &msg);
1322                     tmpl_hdr = (struct ipfix_template_record_header*)
1323                         ((uint8_t*)dp_packet_data(&msg) + tmpl_hdr_offset);
1324                     tmpl_hdr->field_count = htons(field_count);
1325                 }
1326             }
1327         }
1328     }
1329
1330     /* Send template message. */
1331     error_pkts = ipfix_send_template_msg(exporter->collectors, &msg, set_hdr_offset);
1332     tx_errors += error_pkts;
1333     tx_packets += collectors_count(exporter->collectors) - error_pkts;
1334
1335     exporter->stats.tx_pkts += tx_packets;
1336     exporter->stats.tx_errors += tx_errors;
1337
1338     /* XXX: Add Options Template Sets, at least to define a Flow Keys
1339      * Option Template. */
1340
1341 }
1342
1343 static inline uint32_t
1344 ipfix_hash_flow_key(const struct ipfix_flow_key *flow_key, uint32_t basis)
1345 {
1346     uint32_t hash;
1347     hash = hash_int(flow_key->obs_domain_id, basis);
1348     hash = hash_int(flow_key->template_id, hash);
1349     hash = hash_bytes(flow_key->flow_key_msg_part,
1350                       flow_key->flow_key_msg_part_size, hash);
1351     return hash;
1352 }
1353
1354 static bool
1355 ipfix_flow_key_equal(const struct ipfix_flow_key *a,
1356                      const struct ipfix_flow_key *b)
1357 {
1358     /* The template ID determines the flow key size, so not need to
1359      * compare it. */
1360     return (a->obs_domain_id == b->obs_domain_id
1361             && a->template_id == b->template_id
1362             && memcmp(a->flow_key_msg_part, b->flow_key_msg_part,
1363                       a->flow_key_msg_part_size) == 0);
1364 }
1365
1366 static struct ipfix_flow_cache_entry*
1367 ipfix_cache_find_entry(const struct dpif_ipfix_exporter *exporter,
1368                        const struct ipfix_flow_key *flow_key)
1369 {
1370     struct ipfix_flow_cache_entry *entry;
1371
1372     HMAP_FOR_EACH_WITH_HASH (entry, flow_key_map_node,
1373                              ipfix_hash_flow_key(flow_key, 0),
1374                              &exporter->cache_flow_key_map) {
1375         if (ipfix_flow_key_equal(&entry->flow_key, flow_key)) {
1376             return entry;
1377         }
1378     }
1379
1380     return NULL;
1381 }
1382
1383 static bool
1384 ipfix_cache_next_timeout_msec(const struct dpif_ipfix_exporter *exporter,
1385                               long long int *next_timeout_msec)
1386 {
1387     struct ipfix_flow_cache_entry *entry;
1388
1389     LIST_FOR_EACH (entry, cache_flow_start_timestamp_list_node,
1390                    &exporter->cache_flow_start_timestamp_list) {
1391         *next_timeout_msec = entry->flow_start_timestamp_usec / 1000LL
1392             + 1000LL * exporter->cache_active_timeout;
1393         return true;
1394     }
1395
1396     return false;
1397 }
1398
1399 static void
1400 ipfix_cache_aggregate_entries(struct ipfix_flow_cache_entry *from_entry,
1401                               struct ipfix_flow_cache_entry *to_entry)
1402 {
1403     uint64_t *to_start, *to_end, *from_start, *from_end;
1404     uint16_t *to_min_len, *to_max_len, *from_min_len, *from_max_len;
1405
1406     to_start = &to_entry->flow_start_timestamp_usec;
1407     to_end = &to_entry->flow_end_timestamp_usec;
1408     from_start = &from_entry->flow_start_timestamp_usec;
1409     from_end = &from_entry->flow_end_timestamp_usec;
1410
1411     if (*to_start > *from_start) {
1412         *to_start = *from_start;
1413     }
1414     if (*to_end < *from_end) {
1415         *to_end = *from_end;
1416     }
1417
1418     to_entry->packet_delta_count += from_entry->packet_delta_count;
1419     to_entry->layer2_octet_delta_count += from_entry->layer2_octet_delta_count;
1420
1421     to_entry->octet_delta_count += from_entry->octet_delta_count;
1422     to_entry->octet_delta_sum_of_squares +=
1423         from_entry->octet_delta_sum_of_squares;
1424
1425     to_min_len = &to_entry->minimum_ip_total_length;
1426     to_max_len = &to_entry->maximum_ip_total_length;
1427     from_min_len = &from_entry->minimum_ip_total_length;
1428     from_max_len = &from_entry->maximum_ip_total_length;
1429
1430     if (!*to_min_len || (*from_min_len && *to_min_len > *from_min_len)) {
1431         *to_min_len = *from_min_len;
1432     }
1433     if (*to_max_len < *from_max_len) {
1434         *to_max_len = *from_max_len;
1435     }
1436 }
1437
1438 /* Get statistics */
1439 static void
1440 ipfix_get_stats__(const struct dpif_ipfix_exporter *exporter,
1441                   ofproto_ipfix_stats *stats)
1442 {
1443     memset(stats, 0xff, sizeof *stats);
1444
1445     if (!exporter) {
1446         return;
1447     }
1448
1449     *stats = exporter->stats;
1450 }
1451
1452 static void
1453 ipfix_get_bridge_stats(const struct dpif_ipfix_bridge_exporter *exporter,
1454                        ofproto_ipfix_stats *stats)
1455 {
1456     ipfix_get_stats__(&exporter->exporter, stats);
1457 }
1458
1459 static void
1460 ipfix_get_flow_stats(const struct dpif_ipfix_flow_exporter *exporter,
1461                      ofproto_ipfix_stats *stats)
1462 {
1463     ipfix_get_stats__(&exporter->exporter, stats);
1464     stats->collector_set_id = exporter->options->collector_set_id;
1465 }
1466
1467 int
1468 dpif_ipfix_get_stats(const struct dpif_ipfix *di,
1469                      bool bridge_ipfix,
1470                      struct ovs_list *replies)
1471     OVS_EXCLUDED(mutex)
1472 {
1473     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
1474     struct ofputil_ipfix_stats ois;
1475
1476     ovs_mutex_lock(&mutex);
1477     if (bridge_ipfix) {
1478         if (!di->bridge_exporter.options) {
1479             ovs_mutex_unlock(&mutex);
1480             return OFPERR_NXST_NOT_CONFIGURED;
1481         }
1482
1483         ipfix_get_bridge_stats(&di->bridge_exporter, &ois);
1484         ofputil_append_ipfix_stat(replies, &ois);
1485     } else {
1486         if (hmap_count(&di->flow_exporter_map) == 0) {
1487             ovs_mutex_unlock(&mutex);
1488             return OFPERR_NXST_NOT_CONFIGURED;
1489         }
1490
1491         HMAP_FOR_EACH (flow_exporter_node, node,
1492                        &di->flow_exporter_map) {
1493             ipfix_get_flow_stats(&flow_exporter_node->exporter, &ois);
1494             ofputil_append_ipfix_stat(replies, &ois);
1495         }
1496     }
1497     ovs_mutex_unlock(&mutex);
1498
1499     return 0;
1500 }
1501
1502 /* Update partial ipfix stats */
1503 static void
1504 ipfix_update_stats(struct dpif_ipfix_exporter *exporter,
1505                    bool new_flow,
1506                    size_t current_flows,
1507                    enum ipfix_sampled_packet_type sampled_pkt_type)
1508 {
1509     if (new_flow) {
1510         exporter->stats.total_flows++;
1511         exporter->stats.current_flows = current_flows;
1512     }
1513     exporter->stats.pkts++;
1514
1515     switch (sampled_pkt_type) {
1516     case IPFIX_SAMPLED_PKT_IPV4_OK:
1517         exporter->stats.ipv4_pkts++;
1518         break;
1519     case IPFIX_SAMPLED_PKT_IPV6_OK:
1520         exporter->stats.ipv6_pkts++;
1521         break;
1522     case IPFIX_SAMPLED_PKT_IPV4_ERROR:
1523         exporter->stats.ipv4_error_pkts++;
1524         exporter->stats.error_pkts++;
1525         break;
1526     case IPFIX_SAMPLED_PKT_IPV6_ERROR:
1527         exporter->stats.ipv6_error_pkts++;
1528         exporter->stats.error_pkts++;
1529         break;
1530     case IPFIX_SAMPLED_PKT_UNKNOWN:
1531         exporter->stats.error_pkts++;
1532         break;
1533     case IPFIX_SAMPLED_PKT_OTHERS:
1534     default:
1535         break;
1536     }
1537 }
1538
1539 /* Add an entry into a flow cache.  The entry is either aggregated into
1540  * an existing entry with the same flow key and free()d, or it is
1541  * inserted into the cache. And IPFIX stats will be updated */
1542 static void
1543 ipfix_cache_update(struct dpif_ipfix_exporter *exporter,
1544                    struct ipfix_flow_cache_entry *entry,
1545                    enum ipfix_sampled_packet_type sampled_pkt_type)
1546 {
1547     struct ipfix_flow_cache_entry *old_entry;
1548     size_t current_flows = 0;
1549
1550     old_entry = ipfix_cache_find_entry(exporter, &entry->flow_key);
1551
1552     if (old_entry == NULL) {
1553         hmap_insert(&exporter->cache_flow_key_map, &entry->flow_key_map_node,
1554                     ipfix_hash_flow_key(&entry->flow_key, 0));
1555
1556         /* As the latest entry added into the cache, it should
1557          * logically have the highest flow_start_timestamp_usec, so
1558          * append it at the tail. */
1559         ovs_list_push_back(&exporter->cache_flow_start_timestamp_list,
1560                        &entry->cache_flow_start_timestamp_list_node);
1561
1562         /* Enforce exporter->cache_max_flows limit. */
1563         current_flows = hmap_count(&exporter->cache_flow_key_map);
1564         ipfix_update_stats(exporter, true, current_flows, sampled_pkt_type);
1565         if (current_flows > exporter->cache_max_flows) {
1566             dpif_ipfix_cache_expire_now(exporter, false);
1567         }
1568     } else {
1569         ipfix_cache_aggregate_entries(entry, old_entry);
1570         free(entry);
1571         ipfix_update_stats(exporter, false, current_flows, sampled_pkt_type);
1572     }
1573 }
1574
1575 static enum ipfix_sampled_packet_type
1576 ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
1577                        const struct dp_packet *packet, const struct flow *flow,
1578                        uint64_t packet_delta_count, uint32_t obs_domain_id,
1579                        uint32_t obs_point_id, odp_port_t output_odp_port,
1580                        const struct dpif_ipfix_port *tunnel_port,
1581                        const struct flow_tnl *tunnel_key)
1582 {
1583     struct ipfix_flow_key *flow_key;
1584     struct dp_packet msg;
1585     enum ipfix_proto_l2 l2;
1586     enum ipfix_proto_l3 l3;
1587     enum ipfix_proto_l4 l4;
1588     enum ipfix_proto_tunnel tunnel = IPFIX_PROTO_NOT_TUNNELED;
1589     enum ipfix_sampled_packet_type sampled_pkt_type = IPFIX_SAMPLED_PKT_UNKNOWN;
1590     uint8_t ethernet_header_length;
1591     uint16_t ethernet_total_length;
1592
1593     flow_key = &entry->flow_key;
1594     dp_packet_use_stub(&msg, flow_key->flow_key_msg_part,
1595                        sizeof flow_key->flow_key_msg_part);
1596
1597     /* Choose the right template ID matching the protocols in the
1598      * sampled packet. */
1599     l2 = (flow->vlan_tci == 0) ? IPFIX_PROTO_L2_ETH : IPFIX_PROTO_L2_VLAN;
1600
1601     switch(ntohs(flow->dl_type)) {
1602     case ETH_TYPE_IP:
1603         l3 = IPFIX_PROTO_L3_IPV4;
1604         switch(flow->nw_proto) {
1605         case IPPROTO_TCP:
1606         case IPPROTO_UDP:
1607         case IPPROTO_SCTP:
1608             l4 = IPFIX_PROTO_L4_TCP_UDP_SCTP;
1609             sampled_pkt_type = IPFIX_SAMPLED_PKT_IPV4_OK;
1610             break;
1611         case IPPROTO_ICMP:
1612             l4 = IPFIX_PROTO_L4_ICMP;
1613             sampled_pkt_type = IPFIX_SAMPLED_PKT_IPV4_OK;
1614             break;
1615         default:
1616             l4 = IPFIX_PROTO_L4_UNKNOWN;
1617             sampled_pkt_type = IPFIX_SAMPLED_PKT_IPV4_ERROR;
1618         }
1619         break;
1620     case ETH_TYPE_IPV6:
1621         l3 = IPFIX_PROTO_L3_IPV6;
1622         switch(flow->nw_proto) {
1623         case IPPROTO_TCP:
1624         case IPPROTO_UDP:
1625         case IPPROTO_SCTP:
1626             l4 = IPFIX_PROTO_L4_TCP_UDP_SCTP;
1627             sampled_pkt_type = IPFIX_SAMPLED_PKT_IPV6_OK;
1628             break;
1629         case IPPROTO_ICMPV6:
1630             l4 = IPFIX_PROTO_L4_ICMP;
1631             sampled_pkt_type = IPFIX_SAMPLED_PKT_IPV6_OK;
1632             break;
1633         default:
1634             l4 = IPFIX_PROTO_L4_UNKNOWN;
1635             sampled_pkt_type = IPFIX_SAMPLED_PKT_IPV6_ERROR;
1636         }
1637         break;
1638     default:
1639         l3 = IPFIX_PROTO_L3_UNKNOWN;
1640         l4 = IPFIX_PROTO_L4_UNKNOWN;
1641         sampled_pkt_type = IPFIX_SAMPLED_PKT_OTHERS;
1642     }
1643
1644     if (tunnel_port && tunnel_key) {
1645        tunnel = IPFIX_PROTO_TUNNELED;
1646     }
1647
1648     flow_key->obs_domain_id = obs_domain_id;
1649     flow_key->template_id = ipfix_get_template_id(l2, l3, l4, tunnel);
1650
1651     /* The fields defined in the ipfix_data_record_* structs and sent
1652      * below must match exactly the templates defined in
1653      * ipfix_define_template_fields. */
1654
1655     ethernet_header_length = (l2 == IPFIX_PROTO_L2_VLAN)
1656         ? VLAN_ETH_HEADER_LEN : ETH_HEADER_LEN;
1657     ethernet_total_length = dp_packet_size(packet);
1658
1659     /* Common Ethernet entities. */
1660     {
1661         struct ipfix_data_record_flow_key_common *data_common;
1662
1663         data_common = dp_packet_put_zeros(&msg, sizeof *data_common);
1664         data_common->observation_point_id = htonl(obs_point_id);
1665         data_common->flow_direction =
1666             (output_odp_port == ODPP_NONE) ? INGRESS_FLOW : EGRESS_FLOW;
1667         data_common->source_mac_address = flow->dl_src;
1668         data_common->destination_mac_address = flow->dl_dst;
1669         data_common->ethernet_type = flow->dl_type;
1670         data_common->ethernet_header_length = ethernet_header_length;
1671     }
1672
1673     if (l2 == IPFIX_PROTO_L2_VLAN) {
1674         struct ipfix_data_record_flow_key_vlan *data_vlan;
1675         uint16_t vlan_id = vlan_tci_to_vid(flow->vlan_tci);
1676         uint8_t priority = vlan_tci_to_pcp(flow->vlan_tci);
1677
1678         data_vlan = dp_packet_put_zeros(&msg, sizeof *data_vlan);
1679         data_vlan->vlan_id = htons(vlan_id);
1680         data_vlan->dot1q_vlan_id = htons(vlan_id);
1681         data_vlan->dot1q_priority = priority;
1682     }
1683
1684     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1685         struct ipfix_data_record_flow_key_ip *data_ip;
1686
1687         data_ip = dp_packet_put_zeros(&msg, sizeof *data_ip);
1688         data_ip->ip_version = (l3 == IPFIX_PROTO_L3_IPV4) ? 4 : 6;
1689         data_ip->ip_ttl = flow->nw_ttl;
1690         data_ip->protocol_identifier = flow->nw_proto;
1691         data_ip->ip_diff_serv_code_point = flow->nw_tos >> 2;
1692         data_ip->ip_precedence = flow->nw_tos >> 5;
1693         data_ip->ip_class_of_service = flow->nw_tos;
1694
1695         if (l3 == IPFIX_PROTO_L3_IPV4) {
1696             struct ipfix_data_record_flow_key_ipv4 *data_ipv4;
1697
1698             data_ipv4 = dp_packet_put_zeros(&msg, sizeof *data_ipv4);
1699             data_ipv4->source_ipv4_address = flow->nw_src;
1700             data_ipv4->destination_ipv4_address = flow->nw_dst;
1701         } else {  /* l3 == IPFIX_PROTO_L3_IPV6 */
1702             struct ipfix_data_record_flow_key_ipv6 *data_ipv6;
1703
1704             data_ipv6 = dp_packet_put_zeros(&msg, sizeof *data_ipv6);
1705             memcpy(data_ipv6->source_ipv6_address, &flow->ipv6_src,
1706                    sizeof flow->ipv6_src);
1707             memcpy(data_ipv6->destination_ipv6_address, &flow->ipv6_dst,
1708                    sizeof flow->ipv6_dst);
1709             data_ipv6->flow_label_ipv6 = flow->ipv6_label;
1710         }
1711     }
1712
1713     if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1714         struct ipfix_data_record_flow_key_transport *data_transport;
1715
1716         data_transport = dp_packet_put_zeros(&msg, sizeof *data_transport);
1717         data_transport->source_transport_port = flow->tp_src;
1718         data_transport->destination_transport_port = flow->tp_dst;
1719     } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1720         struct ipfix_data_record_flow_key_icmp *data_icmp;
1721
1722         data_icmp = dp_packet_put_zeros(&msg, sizeof *data_icmp);
1723         data_icmp->icmp_type = ntohs(flow->tp_src) & 0xff;
1724         data_icmp->icmp_code = ntohs(flow->tp_dst) & 0xff;
1725     }
1726
1727     if (tunnel == IPFIX_PROTO_TUNNELED) {
1728         struct ipfix_data_record_flow_key_tunnel *data_tunnel;
1729         const uint8_t *tun_id;
1730
1731         data_tunnel = dp_packet_put_zeros(&msg, sizeof *data_tunnel +
1732                                              tunnel_port->tunnel_key_length);
1733         data_tunnel->tunnel_source_ipv4_address = tunnel_key->ip_src;
1734         data_tunnel->tunnel_destination_ipv4_address = tunnel_key->ip_dst;
1735         /* The tunnel_protocol_identifier is from tunnel_proto array, which
1736          * contains protocol_identifiers of each tunnel type.
1737          * For the tunnel type on the top of IPSec, which uses the protocol
1738          * identifier of the upper tunnel type is used, the tcp_src and tcp_dst
1739          * are decided based on the protocol identifiers.
1740          * E.g:
1741          * The protocol identifier of DPIF_IPFIX_TUNNEL_IPSEC_GRE is IPPROTO_GRE,
1742          * and both tp_src and tp_dst are zero.
1743          */
1744         data_tunnel->tunnel_protocol_identifier =
1745             tunnel_protocol[tunnel_port->tunnel_type];
1746         data_tunnel->tunnel_source_transport_port = tunnel_key->tp_src;
1747         data_tunnel->tunnel_destination_transport_port = tunnel_key->tp_dst;
1748         data_tunnel->tunnel_type = tunnel_port->tunnel_type;
1749         data_tunnel->tunnel_key_length = tunnel_port->tunnel_key_length;
1750         /* tun_id is in network order, and tunnel key is in low bits. */
1751         tun_id = (const uint8_t *) &tunnel_key->tun_id;
1752         memcpy(data_tunnel->tunnel_key,
1753                &tun_id[8 - tunnel_port->tunnel_key_length],
1754                tunnel_port->tunnel_key_length);
1755     }
1756
1757     flow_key->flow_key_msg_part_size = dp_packet_size(&msg);
1758
1759     {
1760         struct timeval now;
1761         uint64_t layer2_octet_delta_count;
1762
1763         /* Calculate the total matched octet count by considering as
1764          * an approximation that all matched packets have the same
1765          * length. */
1766         layer2_octet_delta_count = packet_delta_count * ethernet_total_length;
1767
1768         xgettimeofday(&now);
1769         entry->flow_end_timestamp_usec = now.tv_usec + 1000000LL * now.tv_sec;
1770         entry->flow_start_timestamp_usec = entry->flow_end_timestamp_usec;
1771         entry->packet_delta_count = packet_delta_count;
1772         entry->layer2_octet_delta_count = layer2_octet_delta_count;
1773     }
1774
1775     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1776         uint16_t ip_total_length =
1777             ethernet_total_length - ethernet_header_length;
1778         uint64_t octet_delta_count;
1779
1780         /* Calculate the total matched octet count by considering as
1781          * an approximation that all matched packets have the same
1782          * length. */
1783         octet_delta_count = packet_delta_count * ip_total_length;
1784
1785         entry->octet_delta_count = octet_delta_count;
1786         entry->octet_delta_sum_of_squares = octet_delta_count * ip_total_length;
1787         entry->minimum_ip_total_length = ip_total_length;
1788         entry->maximum_ip_total_length = ip_total_length;
1789     } else {
1790         entry->octet_delta_sum_of_squares = 0;
1791         entry->minimum_ip_total_length = 0;
1792         entry->maximum_ip_total_length = 0;
1793     }
1794
1795     return sampled_pkt_type;
1796 }
1797
1798 /* Send each single data record in its own data set, to simplify the
1799  * implementation by avoiding having to group record by template ID
1800  * before sending. */
1801 static void
1802 ipfix_put_data_set(uint32_t export_time_sec,
1803                    struct ipfix_flow_cache_entry *entry,
1804                    enum ipfix_flow_end_reason flow_end_reason,
1805                    const char *virtual_obs_id,
1806                    uint8_t virtual_obs_len,
1807                    struct dp_packet *msg)
1808 {
1809     size_t set_hdr_offset;
1810     struct ipfix_set_header *set_hdr;
1811
1812     set_hdr_offset = dp_packet_size(msg);
1813
1814     /* Put a Data Set. */
1815     set_hdr = dp_packet_put_zeros(msg, sizeof *set_hdr);
1816     set_hdr->set_id = htons(entry->flow_key.template_id);
1817
1818     /* Copy the flow key part of the data record. */
1819
1820     dp_packet_put(msg, entry->flow_key.flow_key_msg_part,
1821                entry->flow_key.flow_key_msg_part_size);
1822
1823     /* Export virtual observation ID. */
1824     if (virtual_obs_id) {
1825         dp_packet_put(msg, &virtual_obs_len, sizeof(virtual_obs_len));
1826         dp_packet_put(msg, virtual_obs_id, virtual_obs_len);
1827     }
1828
1829     /* Put the non-key part of the data record. */
1830
1831     {
1832         struct ipfix_data_record_aggregated_common *data_aggregated_common;
1833         uint64_t export_time_usec, flow_start_delta_usec, flow_end_delta_usec;
1834
1835         /* Calculate the negative deltas relative to the export time
1836          * in seconds sent in the header, not the exact export
1837          * time. */
1838         export_time_usec = 1000000LL * export_time_sec;
1839         flow_start_delta_usec = export_time_usec
1840             - entry->flow_start_timestamp_usec;
1841         flow_end_delta_usec = export_time_usec
1842             - entry->flow_end_timestamp_usec;
1843
1844         data_aggregated_common = dp_packet_put_zeros(
1845             msg, sizeof *data_aggregated_common);
1846         data_aggregated_common->flow_start_delta_microseconds = htonl(
1847             flow_start_delta_usec);
1848         data_aggregated_common->flow_end_delta_microseconds = htonl(
1849             flow_end_delta_usec);
1850         data_aggregated_common->packet_delta_count = htonll(
1851             entry->packet_delta_count);
1852         data_aggregated_common->layer2_octet_delta_count = htonll(
1853             entry->layer2_octet_delta_count);
1854         data_aggregated_common->flow_end_reason = flow_end_reason;
1855     }
1856
1857     if (entry->octet_delta_sum_of_squares) {  /* IP packet. */
1858         struct ipfix_data_record_aggregated_ip *data_aggregated_ip;
1859
1860         data_aggregated_ip = dp_packet_put_zeros(
1861             msg, sizeof *data_aggregated_ip);
1862         data_aggregated_ip->octet_delta_count = htonll(
1863             entry->octet_delta_count);
1864         data_aggregated_ip->octet_delta_sum_of_squares = htonll(
1865             entry->octet_delta_sum_of_squares);
1866         data_aggregated_ip->minimum_ip_total_length = htonll(
1867             entry->minimum_ip_total_length);
1868         data_aggregated_ip->maximum_ip_total_length = htonll(
1869             entry->maximum_ip_total_length);
1870     }
1871
1872     set_hdr = (struct ipfix_set_header*)((uint8_t*)dp_packet_data(msg) + set_hdr_offset);
1873     set_hdr->length = htons(dp_packet_size(msg) - set_hdr_offset);
1874 }
1875
1876 /* Send an IPFIX message with a single data record. */
1877 static void
1878 ipfix_send_data_msg(struct dpif_ipfix_exporter *exporter,
1879                     uint32_t export_time_sec,
1880                     struct ipfix_flow_cache_entry *entry,
1881                     enum ipfix_flow_end_reason flow_end_reason)
1882 {
1883     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
1884     struct dp_packet msg;
1885     size_t tx_errors;
1886
1887     dp_packet_use_stub(&msg, msg_stub, sizeof msg_stub);
1888
1889     ipfix_init_header(export_time_sec, exporter->seq_number++,
1890                       entry->flow_key.obs_domain_id, &msg);
1891     ipfix_put_data_set(export_time_sec, entry, flow_end_reason,
1892                        exporter->virtual_obs_id, exporter->virtual_obs_len,
1893                        &msg);
1894     tx_errors = ipfix_send_msg(exporter->collectors, &msg);
1895
1896     dp_packet_uninit(&msg);
1897
1898     exporter->stats.current_flows--;
1899     exporter->stats.tx_pkts += collectors_count(exporter->collectors) - tx_errors;
1900     exporter->stats.tx_errors += tx_errors;
1901 }
1902
1903 static void
1904 dpif_ipfix_sample(struct dpif_ipfix_exporter *exporter,
1905                   const struct dp_packet *packet, const struct flow *flow,
1906                   uint64_t packet_delta_count, uint32_t obs_domain_id,
1907                   uint32_t obs_point_id, odp_port_t output_odp_port,
1908                   const struct dpif_ipfix_port *tunnel_port,
1909                   const struct flow_tnl *tunnel_key)
1910 {
1911     struct ipfix_flow_cache_entry *entry;
1912     enum ipfix_sampled_packet_type sampled_packet_type;
1913
1914     /* Create a flow cache entry from the sample. */
1915     entry = xmalloc(sizeof *entry);
1916     sampled_packet_type = ipfix_cache_entry_init(entry, packet,
1917                                                  flow, packet_delta_count,
1918                                                  obs_domain_id, obs_point_id,
1919                                                  output_odp_port, tunnel_port,
1920                                                  tunnel_key);
1921     ipfix_cache_update(exporter, entry, sampled_packet_type);
1922 }
1923
1924 static bool
1925 bridge_exporter_enabled(struct dpif_ipfix *di)
1926 {
1927     return di->bridge_exporter.probability > 0;
1928 }
1929
1930 void
1931 dpif_ipfix_bridge_sample(struct dpif_ipfix *di, const struct dp_packet *packet,
1932                          const struct flow *flow,
1933                          odp_port_t input_odp_port, odp_port_t output_odp_port,
1934                          const struct flow_tnl *output_tunnel_key)
1935     OVS_EXCLUDED(mutex)
1936 {
1937     uint64_t packet_delta_count;
1938     const struct flow_tnl *tunnel_key = NULL;
1939     struct dpif_ipfix_port * tunnel_port = NULL;
1940
1941     ovs_mutex_lock(&mutex);
1942     if (!bridge_exporter_enabled(di)) {
1943         ovs_mutex_unlock(&mutex);
1944         return;
1945     }
1946
1947     /* Skip BFD packets:
1948      * Bidirectional Forwarding Detection(BFD) packets are for monitoring
1949      * the tunnel link status and consumed by ovs itself. No need to
1950      * smaple them.
1951      * CF  IETF RFC 5881, BFD control packet is the UDP packet with
1952      * destination port 3784, and BFD echo packet is the UDP packet with
1953      * destination port 3785.
1954      */
1955     if (is_ip_any(flow) &&
1956         flow->nw_proto == IPPROTO_UDP &&
1957         (flow->tp_dst == htons(BFD_CONTROL_DEST_PORT) ||
1958          flow->tp_dst == htons(BFD_ECHO_DEST_PORT))) {
1959         ovs_mutex_unlock(&mutex);
1960         return;
1961     }
1962
1963     /* Use the sampling probability as an approximation of the number
1964      * of matched packets. */
1965     packet_delta_count = UINT32_MAX / di->bridge_exporter.probability;
1966     if (di->bridge_exporter.options->enable_tunnel_sampling) {
1967         if (output_odp_port == ODPP_NONE && flow->tunnel.ip_dst) {
1968             /* Input tunnel. */
1969             tunnel_key = &flow->tunnel;
1970             tunnel_port = dpif_ipfix_find_port(di, input_odp_port);
1971         }
1972         if (output_odp_port != ODPP_NONE && output_tunnel_key) {
1973             /* Output tunnel, output_tunnel_key must be valid. */
1974             tunnel_key = output_tunnel_key;
1975             tunnel_port = dpif_ipfix_find_port(di, output_odp_port);
1976         }
1977     }
1978
1979     dpif_ipfix_sample(&di->bridge_exporter.exporter, packet, flow,
1980                       packet_delta_count,
1981                       di->bridge_exporter.options->obs_domain_id,
1982                       di->bridge_exporter.options->obs_point_id,
1983                       output_odp_port, tunnel_port, tunnel_key);
1984     ovs_mutex_unlock(&mutex);
1985 }
1986
1987 void
1988 dpif_ipfix_flow_sample(struct dpif_ipfix *di, const struct dp_packet *packet,
1989                        const struct flow *flow,
1990                        const union user_action_cookie *cookie,
1991                        odp_port_t input_odp_port,
1992                        const struct flow_tnl *output_tunnel_key)
1993     OVS_EXCLUDED(mutex)
1994 {
1995     struct dpif_ipfix_flow_exporter_map_node *node;
1996     const struct flow_tnl *tunnel_key = NULL;
1997     struct dpif_ipfix_port * tunnel_port = NULL;
1998     odp_port_t output_odp_port = cookie->flow_sample.output_odp_port;
1999     uint32_t collector_set_id = cookie->flow_sample.collector_set_id;
2000     uint16_t probability = cookie->flow_sample.probability;
2001
2002     /* Use the sampling probability as an approximation of the number
2003      * of matched packets. */
2004     uint64_t packet_delta_count = USHRT_MAX / probability;
2005
2006     ovs_mutex_lock(&mutex);
2007     node = dpif_ipfix_find_flow_exporter_map_node(di, collector_set_id);
2008     if (node) {
2009         if (node->exporter.options->enable_tunnel_sampling) {
2010             if (output_odp_port == ODPP_NONE && flow->tunnel.ip_dst) {
2011                 /* Input tunnel. */
2012                 tunnel_key = &flow->tunnel;
2013                 tunnel_port = dpif_ipfix_find_port(di, input_odp_port);
2014             }
2015             if (output_odp_port != ODPP_NONE && output_tunnel_key) {
2016                 /* Output tunnel, output_tunnel_key must be valid. */
2017                 tunnel_key = output_tunnel_key;
2018                 tunnel_port = dpif_ipfix_find_port(di, output_odp_port);
2019             }
2020         }
2021
2022         dpif_ipfix_sample(&node->exporter.exporter, packet, flow,
2023                           packet_delta_count,
2024                           cookie->flow_sample.obs_domain_id,
2025                           cookie->flow_sample.obs_point_id,
2026                           output_odp_port, tunnel_port, tunnel_key);
2027     }
2028     ovs_mutex_unlock(&mutex);
2029 }
2030
2031 static void
2032 dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *exporter,
2033                         bool forced_end, const uint64_t export_time_usec,
2034                         const uint32_t export_time_sec)
2035 {
2036     struct ipfix_flow_cache_entry *entry, *next_entry;
2037     uint64_t max_flow_start_timestamp_usec;
2038     bool template_msg_sent = false;
2039     enum ipfix_flow_end_reason flow_end_reason;
2040
2041     if (ovs_list_is_empty(&exporter->cache_flow_start_timestamp_list)) {
2042         return;
2043     }
2044
2045     max_flow_start_timestamp_usec = export_time_usec -
2046         1000000LL * exporter->cache_active_timeout;
2047
2048     LIST_FOR_EACH_SAFE (entry, next_entry, cache_flow_start_timestamp_list_node,
2049                         &exporter->cache_flow_start_timestamp_list) {
2050         if (forced_end) {
2051             flow_end_reason = FORCED_END;
2052         } else if (entry->flow_start_timestamp_usec
2053                    <= max_flow_start_timestamp_usec) {
2054             flow_end_reason = ACTIVE_TIMEOUT;
2055         } else if (hmap_count(&exporter->cache_flow_key_map)
2056                    > exporter->cache_max_flows) {
2057             /* Enforce exporter->cache_max_flows. */
2058             flow_end_reason = LACK_OF_RESOURCES;
2059         } else {
2060             /* Remaining flows haven't expired yet. */
2061             break;
2062         }
2063
2064         ovs_list_remove(&entry->cache_flow_start_timestamp_list_node);
2065         hmap_remove(&exporter->cache_flow_key_map,
2066                     &entry->flow_key_map_node);
2067
2068         if (!template_msg_sent
2069             && (exporter->last_template_set_time + IPFIX_TEMPLATE_INTERVAL)
2070                 <= export_time_sec) {
2071             ipfix_send_template_msgs(exporter, export_time_sec,
2072                                      entry->flow_key.obs_domain_id);
2073             exporter->last_template_set_time = export_time_sec;
2074             template_msg_sent = true;
2075         }
2076
2077         /* XXX: Group multiple data records for the same obs domain id
2078          * into the same message. */
2079         ipfix_send_data_msg(exporter, export_time_sec, entry, flow_end_reason);
2080         free(entry);
2081     }
2082 }
2083
2084 static void
2085 get_export_time_now(uint64_t *export_time_usec, uint32_t *export_time_sec)
2086 {
2087     struct timeval export_time;
2088     xgettimeofday(&export_time);
2089
2090     *export_time_usec = export_time.tv_usec + 1000000LL * export_time.tv_sec;
2091
2092     /* The IPFIX start and end deltas are negative deltas relative to
2093      * the export time, so set the export time 1 second off to
2094      * calculate those deltas. */
2095     if (export_time.tv_usec == 0) {
2096         *export_time_sec = export_time.tv_sec;
2097     } else {
2098         *export_time_sec = export_time.tv_sec + 1;
2099     }
2100 }
2101
2102 static void
2103 dpif_ipfix_cache_expire_now(struct dpif_ipfix_exporter *exporter,
2104                             bool forced_end)
2105 {
2106     uint64_t export_time_usec;
2107     uint32_t export_time_sec;
2108
2109     get_export_time_now(&export_time_usec, &export_time_sec);
2110     dpif_ipfix_cache_expire(exporter, forced_end, export_time_usec,
2111                             export_time_sec);
2112 }
2113
2114 void
2115 dpif_ipfix_run(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
2116 {
2117     uint64_t export_time_usec;
2118     uint32_t export_time_sec;
2119     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
2120
2121     ovs_mutex_lock(&mutex);
2122     get_export_time_now(&export_time_usec, &export_time_sec);
2123     if (bridge_exporter_enabled(di)) {
2124       dpif_ipfix_cache_expire(
2125           &di->bridge_exporter.exporter, false, export_time_usec,
2126           export_time_sec);
2127     }
2128     HMAP_FOR_EACH (flow_exporter_node, node, &di->flow_exporter_map) {
2129         dpif_ipfix_cache_expire(
2130             &flow_exporter_node->exporter.exporter, false, export_time_usec,
2131             export_time_sec);
2132     }
2133     ovs_mutex_unlock(&mutex);
2134 }
2135
2136 void
2137 dpif_ipfix_wait(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
2138 {
2139     long long int next_timeout_msec = LLONG_MAX;
2140     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
2141
2142     ovs_mutex_lock(&mutex);
2143     if (bridge_exporter_enabled(di)) {
2144         if (ipfix_cache_next_timeout_msec(
2145                 &di->bridge_exporter.exporter, &next_timeout_msec)) {
2146             poll_timer_wait_until(next_timeout_msec);
2147         }
2148     }
2149     HMAP_FOR_EACH (flow_exporter_node, node, &di->flow_exporter_map) {
2150         if (ipfix_cache_next_timeout_msec(
2151                 &flow_exporter_node->exporter.exporter, &next_timeout_msec)) {
2152             poll_timer_wait_until(next_timeout_msec);
2153         }
2154     }
2155     ovs_mutex_unlock(&mutex);
2156 }