ofproto/ofproto-dpif-ipfix.c

   1 /*
   2  * Copyright (c) 2012, 2013, 2014 Nicira, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <config.h>
  18 #include "ofproto-dpif-ipfix.h"
  19 #include <sys/time.h>
  20 #include "byte-order.h"
  21 #include "collectors.h"
  22 #include "flow.h"
  23 #include "hash.h"
  24 #include "hmap.h"
  25 #include "list.h"
  26 #include "ofpbuf.h"
  27 #include "ofproto.h"
  28 #include "ofproto-dpif.h"
  29 #include "packets.h"
  30 #include "poll-loop.h"
  31 #include "sset.h"
  32 #include "util.h"
  33 #include "timeval.h"
  34 #include "util.h"
  35 #include "openvswitch/vlog.h"
  36
  37 VLOG_DEFINE_THIS_MODULE(ipfix);
  38
  39 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
  40 static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
  41
  42 /* Cf. IETF RFC 5101 Section 10.3.4. */
  43 #define IPFIX_DEFAULT_COLLECTOR_PORT 4739
  44
  45 /* The standard layer2SegmentId (ID 351) element is included in vDS to send
  46  * the VxLAN tunnel's VNI. It is 64-bit long, the most significant byte is
  47  * used to indicate the type of tunnel (0x01 = VxLAN, 0x02 = GRE) and the three
  48  * least significant bytes hold the value of the layer 2 overlay network
  49  * segment identifier: a 24-bit VxLAN tunnel's VNI or a 24-bit GRE tunnel's
  50  * TNI. This is not compatible with GRE-64, as implemented in OVS, as its
  51  * tunnel IDs are 64-bit.
  52  *
  53  * Two new enterprise information elements are defined which are similar to
  54  * laryerSegmentId but support 64-bit IDs:
  55  *     tunnelType (ID 891) and tunnelKey (ID 892).
  56  *
  57  * The enum dpif_ipfix_tunnel_type is to declare the types supported in the
  58  * tunnelType element.
  59  * The number of ipfix tunnel types includes two reserverd types: 0x04 and 0x06.
  60  */
  61 enum dpif_ipfix_tunnel_type {
  62     DPIF_IPFIX_TUNNEL_UNKNOWN = 0x00,
  63     DPIF_IPFIX_TUNNEL_VXLAN = 0x01,
  64     DPIF_IPFIX_TUNNEL_GRE = 0x02,
  65     DPIF_IPFIX_TUNNEL_LISP = 0x03,
  66     DPIF_IPFIX_TUNNEL_IPSEC_GRE = 0x05,
  67     DPIF_IPFIX_TUNNEL_GENEVE = 0x07,
  68     NUM_DPIF_IPFIX_TUNNEL
  69 };
  70
  71 struct dpif_ipfix_port {
  72     struct hmap_node hmap_node; /* In struct dpif_ipfix's "tunnel_ports" hmap. */
  73     struct ofport *ofport;      /* To retrieve port stats. */
  74     odp_port_t odp_port;
  75     enum dpif_ipfix_tunnel_type tunnel_type;
  76     uint8_t tunnel_key_length;
  77 };
  78
  79 struct dpif_ipfix_exporter {
  80     struct collectors *collectors;
  81     uint32_t seq_number;
  82     time_t last_template_set_time;
  83     struct hmap cache_flow_key_map;  /* ipfix_flow_cache_entry. */
  84     struct ovs_list cache_flow_start_timestamp_list;  /* ipfix_flow_cache_entry. */
  85     uint32_t cache_active_timeout;  /* In seconds. */
  86     uint32_t cache_max_flows;
  87 };
  88
  89 struct dpif_ipfix_bridge_exporter {
  90     struct dpif_ipfix_exporter exporter;
  91     struct ofproto_ipfix_bridge_exporter_options *options;
  92     uint32_t probability;
  93 };
  94
  95 struct dpif_ipfix_flow_exporter {
  96     struct dpif_ipfix_exporter exporter;
  97     struct ofproto_ipfix_flow_exporter_options *options;
  98 };
  99
 100 struct dpif_ipfix_flow_exporter_map_node {
 101     struct hmap_node node;
 102     struct dpif_ipfix_flow_exporter exporter;
 103 };
 104
 105 struct dpif_ipfix {
 106     struct dpif_ipfix_bridge_exporter bridge_exporter;
 107     struct hmap flow_exporter_map;  /* dpif_ipfix_flow_exporter_map_node. */
 108     struct hmap tunnel_ports;       /* Contains "struct dpif_ipfix_port"s.
 109                                      * It makes tunnel port lookups faster in
 110                                      * sampling upcalls. */
 111     struct ovs_refcount ref_cnt;
 112 };
 113
 114 #define IPFIX_VERSION 0x000a
 115
 116 /* When using UDP, IPFIX Template Records must be re-sent regularly.
 117  * The standard default interval is 10 minutes (600 seconds).
 118  * Cf. IETF RFC 5101 Section 10.3.6. */
 119 #define IPFIX_TEMPLATE_INTERVAL 600
 120
 121 /* Cf. IETF RFC 5101 Section 3.1. */
 122 OVS_PACKED(
 123 struct ipfix_header {
 124     ovs_be16 version;  /* IPFIX_VERSION. */
 125     ovs_be16 length;  /* Length in bytes including this header. */
 126     ovs_be32 export_time;  /* Seconds since the epoch. */
 127     ovs_be32 seq_number;  /* Message sequence number. */
 128     ovs_be32 obs_domain_id;  /* Observation Domain ID. */
 129 });
 130 BUILD_ASSERT_DECL(sizeof(struct ipfix_header) == 16);
 131
 132 #define IPFIX_SET_ID_TEMPLATE 2
 133 #define IPFIX_SET_ID_OPTION_TEMPLATE 3
 134
 135 /* Cf. IETF RFC 5101 Section 3.3.2. */
 136 OVS_PACKED(
 137 struct ipfix_set_header {
 138     ovs_be16 set_id;  /* IPFIX_SET_ID_* or valid template ID for Data Sets. */
 139     ovs_be16 length;  /* Length of the set in bytes including header. */
 140 });
 141 BUILD_ASSERT_DECL(sizeof(struct ipfix_set_header) == 4);
 142
 143 /* Alternatives for templates at each layer.  A template is defined by
 144  * a combination of one value for each layer. */
 145 enum ipfix_proto_l2 {
 146     IPFIX_PROTO_L2_ETH = 0,  /* No VLAN. */
 147     IPFIX_PROTO_L2_VLAN,
 148     NUM_IPFIX_PROTO_L2
 149 };
 150 enum ipfix_proto_l3 {
 151     IPFIX_PROTO_L3_UNKNOWN = 0,
 152     IPFIX_PROTO_L3_IPV4,
 153     IPFIX_PROTO_L3_IPV6,
 154     NUM_IPFIX_PROTO_L3
 155 };
 156 enum ipfix_proto_l4 {
 157     IPFIX_PROTO_L4_UNKNOWN = 0,
 158     IPFIX_PROTO_L4_TCP_UDP_SCTP,
 159     IPFIX_PROTO_L4_ICMP,
 160     NUM_IPFIX_PROTO_L4
 161 };
 162 enum ipfix_proto_tunnel {
 163     IPFIX_PROTO_NOT_TUNNELED = 0,
 164     IPFIX_PROTO_TUNNELED,  /* Support gre, lisp and vxlan. */
 165     NUM_IPFIX_PROTO_TUNNEL
 166 };
 167
 168 /* Any Template ID > 255 is usable for Template Records. */
 169 #define IPFIX_TEMPLATE_ID_MIN 256
 170
 171 /* Cf. IETF RFC 5101 Section 3.4.1. */
 172 OVS_PACKED(
 173 struct ipfix_template_record_header {
 174     ovs_be16 template_id;
 175     ovs_be16 field_count;
 176 });
 177 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_record_header) == 4);
 178
 179 enum ipfix_entity_id {
 180 /* standard IPFIX elements */
 181 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ID_##ENUM = ID,
 182 #include "ofproto/ipfix-entities.def"
 183 /* non-standard IPFIX elements */
 184 #define IPFIX_SET_ENTERPRISE(v) (((v) | 0x8000))
 185 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 186     IPFIX_ENTITY_ID_##ENUM = IPFIX_SET_ENTERPRISE(ID),
 187 #include "ofproto/ipfix-enterprise-entities.def"
 188 };
 189
 190 enum ipfix_entity_size {
 191 /* standard IPFIX elements */
 192 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 193 #include "ofproto/ipfix-entities.def"
 194 /* non-standard IPFIX elements */
 195 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 196     IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 197 #include "ofproto/ipfix-enterprise-entities.def"
 198 };
 199
 200 enum ipfix_entity_enterprise {
 201 /* standard IPFIX elements */
 202 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ENTERPRISE_##ENUM = 0,
 203 #include "ofproto/ipfix-entities.def"
 204 /* non-standard IPFIX elements */
 205 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 206     IPFIX_ENTITY_ENTERPRISE_##ENUM = ENTERPRISE,
 207 #include "ofproto/ipfix-enterprise-entities.def"
 208 };
 209
 210 OVS_PACKED(
 211 struct ipfix_template_field_specifier {
 212     ovs_be16 element_id;  /* IPFIX_ENTITY_ID_*. */
 213     ovs_be16 field_length;  /* Length of the field's value, in bytes.
 214                              * For Variable-Length element, it should be 65535.
 215                              */
 216     ovs_be32 enterprise;  /* Enterprise number */
 217 });
 218 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 8);
 219
 220 /* Cf. IETF RFC 5102 Section 5.11.6. */
 221 enum ipfix_flow_direction {
 222     INGRESS_FLOW = 0x00,
 223     EGRESS_FLOW = 0x01
 224 };
 225
 226 /* Part of data record flow key for common metadata and Ethernet entities. */
 227 OVS_PACKED(
 228 struct ipfix_data_record_flow_key_common {
 229     ovs_be32 observation_point_id;  /* OBSERVATION_POINT_ID */
 230     uint8_t flow_direction;  /* FLOW_DIRECTION */
 231     uint8_t source_mac_address[ETH_ADDR_LEN]; /* SOURCE_MAC_ADDRESS */
 232     uint8_t destination_mac_address[ETH_ADDR_LEN]; /* DESTINATION_MAC_ADDRESS */
 233     ovs_be16 ethernet_type;  /* ETHERNET_TYPE */
 234     uint8_t ethernet_header_length;  /* ETHERNET_HEADER_LENGTH */
 235 });
 236 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 20);
 237
 238 /* Part of data record flow key for VLAN entities. */
 239 OVS_PACKED(
 240 struct ipfix_data_record_flow_key_vlan {
 241     ovs_be16 vlan_id;  /* VLAN_ID */
 242     ovs_be16 dot1q_vlan_id;  /* DOT1Q_VLAN_ID */
 243     uint8_t dot1q_priority;  /* DOT1Q_PRIORITY */
 244 });
 245 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_vlan) == 5);
 246
 247 /* Part of data record flow key for IP entities. */
 248 /* XXX: Replace IP_TTL with MINIMUM_TTL and MAXIMUM_TTL? */
 249 OVS_PACKED(
 250 struct ipfix_data_record_flow_key_ip {
 251     uint8_t ip_version;  /* IP_VERSION */
 252     uint8_t ip_ttl;  /* IP_TTL */
 253     uint8_t protocol_identifier;  /* PROTOCOL_IDENTIFIER */
 254     uint8_t ip_diff_serv_code_point;  /* IP_DIFF_SERV_CODE_POINT */
 255     uint8_t ip_precedence;  /* IP_PRECEDENCE */
 256     uint8_t ip_class_of_service;  /* IP_CLASS_OF_SERVICE */
 257 });
 258 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ip) == 6);
 259
 260 /* Part of data record flow key for IPv4 entities. */
 261 OVS_PACKED(
 262 struct ipfix_data_record_flow_key_ipv4 {
 263     ovs_be32 source_ipv4_address;  /* SOURCE_IPV4_ADDRESS */
 264     ovs_be32 destination_ipv4_address;  /* DESTINATION_IPV4_ADDRESS */
 265 });
 266 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ipv4) == 8);
 267
 268 /* Part of data record flow key for IPv6 entities. */
 269 OVS_PACKED(
 270 struct ipfix_data_record_flow_key_ipv6 {
 271     uint8_t source_ipv6_address[16];  /* SOURCE_IPV6_ADDRESS */
 272     uint8_t destination_ipv6_address[16];  /* DESTINATION_IPV6_ADDRESS */
 273     ovs_be32 flow_label_ipv6;  /* FLOW_LABEL_IPV6 */
 274 });
 275 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ipv6) == 36);
 276
 277 /* Part of data record flow key for TCP/UDP/SCTP entities. */
 278 OVS_PACKED(
 279 struct ipfix_data_record_flow_key_transport {
 280     ovs_be16 source_transport_port;  /* SOURCE_TRANSPORT_PORT */
 281     ovs_be16 destination_transport_port;  /* DESTINATION_TRANSPORT_PORT */
 282 });
 283 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_transport) == 4);
 284
 285 /* Part of data record flow key for ICMP entities. */
 286 OVS_PACKED(
 287 struct ipfix_data_record_flow_key_icmp {
 288     uint8_t icmp_type;  /* ICMP_TYPE_IPV4 / ICMP_TYPE_IPV6 */
 289     uint8_t icmp_code;  /* ICMP_CODE_IPV4 / ICMP_CODE_IPV6 */
 290 });
 291 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_icmp) == 2);
 292
 293 /* For the tunnel type that is on the top of IPSec, the protocol identifier
 294  * of the upper tunnel type is used.
 295  */
 296 static uint8_t tunnel_protocol[NUM_DPIF_IPFIX_TUNNEL] = {
 297     0,              /* reserved */
 298     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_VXLAN */
 299     IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_GRE */
 300     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_LISP*/
 301     0          ,    /* reserved */
 302     IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_IPSEC_GRE */
 303     0          ,    /* reserved */
 304     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_GENEVE*/
 305 };
 306
 307 OVS_PACKED(
 308 struct ipfix_data_record_flow_key_tunnel {
 309     ovs_be32 tunnel_source_ipv4_address;  /* TUNNEL_SOURCE_IPV4_ADDRESS */
 310     ovs_be32 tunnel_destination_ipv4_address;  /* TUNNEL_DESTINATION_IPV4_ADDRESS */
 311     uint8_t tunnel_protocol_identifier;  /* TUNNEL_PROTOCOL_IDENTIFIER */
 312     ovs_be16 tunnel_source_transport_port;  /* TUNNEL_SOURCE_TRANSPORT_PORT */
 313     ovs_be16 tunnel_destination_transport_port;  /* TUNNEL_DESTINATION_TRANSPORT_PORT */
 314     uint8_t tunnel_type;  /* TUNNEL_TYPE */
 315     uint8_t tunnel_key_length;  /* length of TUNNEL_KEY */
 316     uint8_t tunnel_key[];  /* data of  TUNNEL_KEY */
 317 });
 318 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_tunnel) == 15);
 319
 320 /* Cf. IETF RFC 5102 Section 5.11.3. */
 321 enum ipfix_flow_end_reason {
 322     IDLE_TIMEOUT = 0x01,
 323     ACTIVE_TIMEOUT = 0x02,
 324     END_OF_FLOW_DETECTED = 0x03,
 325     FORCED_END = 0x04,
 326     LACK_OF_RESOURCES = 0x05
 327 };
 328
 329 /* Part of data record for common aggregated elements. */
 330 OVS_PACKED(
 331 struct ipfix_data_record_aggregated_common {
 332     ovs_be32 flow_start_delta_microseconds; /* FLOW_START_DELTA_MICROSECONDS */
 333     ovs_be32 flow_end_delta_microseconds; /* FLOW_END_DELTA_MICROSECONDS */
 334     ovs_be64 packet_delta_count;  /* PACKET_DELTA_COUNT */
 335     ovs_be64 layer2_octet_delta_count;  /* LAYER2_OCTET_DELTA_COUNT */
 336     uint8_t flow_end_reason;  /* FLOW_END_REASON */
 337 });
 338 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_common) == 25);
 339
 340 /* Part of data record for IP aggregated elements. */
 341 OVS_PACKED(
 342 struct ipfix_data_record_aggregated_ip {
 343     ovs_be64 octet_delta_count;  /* OCTET_DELTA_COUNT */
 344     ovs_be64 octet_delta_sum_of_squares;  /* OCTET_DELTA_SUM_OF_SQUARES */
 345     ovs_be64 minimum_ip_total_length;  /* MINIMUM_IP_TOTAL_LENGTH */
 346     ovs_be64 maximum_ip_total_length;  /* MAXIMUM_IP_TOTAL_LENGTH */
 347 });
 348 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
 349
 350 /*
 351  * support tunnel key for:
 352  * VxLAN: 24-bit VIN,
 353  * GRE: 32- or 64-bit key,
 354  * LISP: 24-bit instance ID
 355  */
 356 #define MAX_TUNNEL_KEY_LEN 8
 357
 358 #define MAX_FLOW_KEY_LEN                                        \
 359     (sizeof(struct ipfix_data_record_flow_key_common)           \
 360      + sizeof(struct ipfix_data_record_flow_key_vlan)           \
 361      + sizeof(struct ipfix_data_record_flow_key_ip)             \
 362      + MAX(sizeof(struct ipfix_data_record_flow_key_ipv4),      \
 363            sizeof(struct ipfix_data_record_flow_key_ipv6))      \
 364      + MAX(sizeof(struct ipfix_data_record_flow_key_icmp),      \
 365            sizeof(struct ipfix_data_record_flow_key_transport)) \
 366      + sizeof(struct ipfix_data_record_flow_key_tunnel)         \
 367      + MAX_TUNNEL_KEY_LEN)
 368
 369 #define MAX_DATA_RECORD_LEN                                 \
 370     (MAX_FLOW_KEY_LEN                                       \
 371      + sizeof(struct ipfix_data_record_aggregated_common)   \
 372      + sizeof(struct ipfix_data_record_aggregated_ip))
 373
 374 /* Max length of a data set.  To simplify the implementation, each
 375  * data record is sent in a separate data set, so each data set
 376  * contains at most one data record. */
 377 #define MAX_DATA_SET_LEN             \
 378     (sizeof(struct ipfix_set_header) \
 379      + MAX_DATA_RECORD_LEN)
 380
 381 /* Max length of an IPFIX message. Arbitrarily set to accommodate low
 382  * MTU. */
 383 #define MAX_MESSAGE_LEN 1024
 384
 385 /* Cache structures. */
 386
 387 /* Flow key. */
 388 struct ipfix_flow_key {
 389     uint32_t obs_domain_id;
 390     uint16_t template_id;
 391     size_t flow_key_msg_part_size;
 392     uint64_t flow_key_msg_part[DIV_ROUND_UP(MAX_FLOW_KEY_LEN, 8)];
 393 };
 394
 395 /* Flow cache entry. */
 396 struct ipfix_flow_cache_entry {
 397     struct hmap_node flow_key_map_node;
 398     struct ovs_list cache_flow_start_timestamp_list_node;
 399     struct ipfix_flow_key flow_key;
 400     /* Common aggregated elements. */
 401     uint64_t flow_start_timestamp_usec;
 402     uint64_t flow_end_timestamp_usec;
 403     uint64_t packet_delta_count;
 404     uint64_t layer2_octet_delta_count;
 405     uint64_t octet_delta_count;
 406     uint64_t octet_delta_sum_of_squares;  /* 0 if not IP. */
 407     uint16_t minimum_ip_total_length;  /* 0 if not IP. */
 408     uint16_t maximum_ip_total_length;  /* 0 if not IP. */
 409 };
 410
 411 static void dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *, bool,
 412                                     const uint64_t, const uint32_t);
 413
 414 static void get_export_time_now(uint64_t *, uint32_t *);
 415
 416 static void dpif_ipfix_cache_expire_now(struct dpif_ipfix_exporter *, bool);
 417
 418 static bool
 419 ofproto_ipfix_bridge_exporter_options_equal(
 420     const struct ofproto_ipfix_bridge_exporter_options *a,
 421     const struct ofproto_ipfix_bridge_exporter_options *b)
 422 {
 423     return (a->obs_domain_id == b->obs_domain_id
 424             && a->obs_point_id == b->obs_point_id
 425             && a->sampling_rate == b->sampling_rate
 426             && a->cache_active_timeout == b->cache_active_timeout
 427             && a->cache_max_flows == b->cache_max_flows
 428             && a->enable_tunnel_sampling == b->enable_tunnel_sampling
 429             && a->enable_input_sampling == b->enable_input_sampling
 430             && a->enable_output_sampling == b->enable_output_sampling
 431             && sset_equals(&a->targets, &b->targets));
 432 }
 433
 434 static struct ofproto_ipfix_bridge_exporter_options *
 435 ofproto_ipfix_bridge_exporter_options_clone(
 436     const struct ofproto_ipfix_bridge_exporter_options *old)
 437 {
 438     struct ofproto_ipfix_bridge_exporter_options *new =
 439         xmemdup(old, sizeof *old);
 440     sset_clone(&new->targets, &old->targets);
 441     return new;
 442 }
 443
 444 static void
 445 ofproto_ipfix_bridge_exporter_options_destroy(
 446     struct ofproto_ipfix_bridge_exporter_options *options)
 447 {
 448     if (options) {
 449         sset_destroy(&options->targets);
 450         free(options);
 451     }
 452 }
 453
 454 static bool
 455 ofproto_ipfix_flow_exporter_options_equal(
 456     const struct ofproto_ipfix_flow_exporter_options *a,
 457     const struct ofproto_ipfix_flow_exporter_options *b)
 458 {
 459     return (a->collector_set_id == b->collector_set_id
 460             && a->cache_active_timeout == b->cache_active_timeout
 461             && a->cache_max_flows == b->cache_max_flows
 462             && sset_equals(&a->targets, &b->targets));
 463 }
 464
 465 static struct ofproto_ipfix_flow_exporter_options *
 466 ofproto_ipfix_flow_exporter_options_clone(
 467     const struct ofproto_ipfix_flow_exporter_options *old)
 468 {
 469     struct ofproto_ipfix_flow_exporter_options *new =
 470         xmemdup(old, sizeof *old);
 471     sset_clone(&new->targets, &old->targets);
 472     return new;
 473 }
 474
 475 static void
 476 ofproto_ipfix_flow_exporter_options_destroy(
 477     struct ofproto_ipfix_flow_exporter_options *options)
 478 {
 479     if (options) {
 480         sset_destroy(&options->targets);
 481         free(options);
 482     }
 483 }
 484
 485 static void
 486 dpif_ipfix_exporter_init(struct dpif_ipfix_exporter *exporter)
 487 {
 488     exporter->collectors = NULL;
 489     exporter->seq_number = 1;
 490     exporter->last_template_set_time = TIME_MIN;
 491     hmap_init(&exporter->cache_flow_key_map);
 492     list_init(&exporter->cache_flow_start_timestamp_list);
 493     exporter->cache_active_timeout = 0;
 494     exporter->cache_max_flows = 0;
 495 }
 496
 497 static void
 498 dpif_ipfix_exporter_clear(struct dpif_ipfix_exporter *exporter)
 499 {
 500     /* Flush the cache with flow end reason "forced end." */
 501     dpif_ipfix_cache_expire_now(exporter, true);
 502
 503     collectors_destroy(exporter->collectors);
 504     exporter->collectors = NULL;
 505     exporter->seq_number = 1;
 506     exporter->last_template_set_time = TIME_MIN;
 507     exporter->cache_active_timeout = 0;
 508     exporter->cache_max_flows = 0;
 509 }
 510
 511 static void
 512 dpif_ipfix_exporter_destroy(struct dpif_ipfix_exporter *exporter)
 513 {
 514     dpif_ipfix_exporter_clear(exporter);
 515     hmap_destroy(&exporter->cache_flow_key_map);
 516 }
 517
 518 static bool
 519 dpif_ipfix_exporter_set_options(struct dpif_ipfix_exporter *exporter,
 520                                 const struct sset *targets,
 521                                 const uint32_t cache_active_timeout,
 522                                 const uint32_t cache_max_flows)
 523 {
 524     collectors_destroy(exporter->collectors);
 525     collectors_create(targets, IPFIX_DEFAULT_COLLECTOR_PORT,
 526                       &exporter->collectors);
 527     if (exporter->collectors == NULL) {
 528         VLOG_WARN_RL(&rl, "no collectors could be initialized, "
 529                      "IPFIX exporter disabled");
 530         dpif_ipfix_exporter_clear(exporter);
 531         return false;
 532     }
 533     exporter->cache_active_timeout = cache_active_timeout;
 534     exporter->cache_max_flows = cache_max_flows;
 535     return true;
 536 }
 537
 538 static struct dpif_ipfix_port *
 539 dpif_ipfix_find_port(const struct dpif_ipfix *di,
 540                      odp_port_t odp_port) OVS_REQUIRES(mutex)
 541 {
 542     struct dpif_ipfix_port *dip;
 543
 544     HMAP_FOR_EACH_IN_BUCKET (dip, hmap_node, hash_odp_port(odp_port),
 545                              &di->tunnel_ports) {
 546         if (dip->odp_port == odp_port) {
 547             return dip;
 548         }
 549     }
 550     return NULL;
 551 }
 552
 553 static void
 554 dpif_ipfix_del_port(struct dpif_ipfix *di,
 555                       struct dpif_ipfix_port *dip)
 556     OVS_REQUIRES(mutex)
 557 {
 558     hmap_remove(&di->tunnel_ports, &dip->hmap_node);
 559     free(dip);
 560 }
 561
 562 void
 563 dpif_ipfix_add_tunnel_port(struct dpif_ipfix *di, struct ofport *ofport,
 564                            odp_port_t odp_port) OVS_EXCLUDED(mutex)
 565 {
 566     struct dpif_ipfix_port *dip;
 567     const char *type;
 568
 569     ovs_mutex_lock(&mutex);
 570     dip = dpif_ipfix_find_port(di, odp_port);
 571     if (dip) {
 572         dpif_ipfix_del_port(di, dip);
 573     }
 574
 575     type = netdev_get_type(ofport->netdev);
 576     if (type == NULL) {
 577         goto out;
 578     }
 579
 580     /* Add to table of tunnel ports. */
 581     dip = xmalloc(sizeof *dip);
 582     dip->ofport = ofport;
 583     dip->odp_port = odp_port;
 584     if (strcmp(type, "gre") == 0) {
 585         /* 32-bit key gre */
 586         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
 587         dip->tunnel_key_length = 4;
 588     } else if (strcmp(type, "gre64") == 0) {
 589         /* 64-bit key gre */
 590         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
 591         dip->tunnel_key_length = 8;
 592     } else if (strcmp(type, "ipsec_gre") == 0) {
 593         /* 32-bit key ipsec_gre */
 594         dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
 595         dip->tunnel_key_length = 4;
 596     } else if (strcmp(type, "ipsec_gre64") == 0) {
 597         /* 64-bit key ipsec_gre */
 598         dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
 599         dip->tunnel_key_length = 8;
 600     } else if (strcmp(type, "vxlan") == 0) {
 601         dip->tunnel_type = DPIF_IPFIX_TUNNEL_VXLAN;
 602         dip->tunnel_key_length = 3;
 603     } else if (strcmp(type, "lisp") == 0) {
 604         dip->tunnel_type = DPIF_IPFIX_TUNNEL_LISP;
 605         dip->tunnel_key_length = 3;
 606     } else if (strcmp(type, "geneve") == 0) {
 607         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GENEVE;
 608         dip->tunnel_key_length = 3;
 609     } else {
 610         free(dip);
 611         goto out;
 612     }
 613     hmap_insert(&di->tunnel_ports, &dip->hmap_node, hash_odp_port(odp_port));
 614
 615 out:
 616     ovs_mutex_unlock(&mutex);
 617 }
 618
 619 void
 620 dpif_ipfix_del_tunnel_port(struct dpif_ipfix *di, odp_port_t odp_port)
 621     OVS_EXCLUDED(mutex)
 622 {
 623     struct dpif_ipfix_port *dip;
 624     ovs_mutex_lock(&mutex);
 625     dip = dpif_ipfix_find_port(di, odp_port);
 626     if (dip) {
 627         dpif_ipfix_del_port(di, dip);
 628     }
 629     ovs_mutex_unlock(&mutex);
 630 }
 631
 632 bool
 633 dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *di, odp_port_t odp_port)
 634     OVS_EXCLUDED(mutex)
 635 {
 636     struct dpif_ipfix_port *dip;
 637     ovs_mutex_lock(&mutex);
 638     dip = dpif_ipfix_find_port(di, odp_port);
 639     ovs_mutex_unlock(&mutex);
 640     return dip != NULL;
 641 }
 642
 643 static void
 644 dpif_ipfix_bridge_exporter_init(struct dpif_ipfix_bridge_exporter *exporter)
 645 {
 646     dpif_ipfix_exporter_init(&exporter->exporter);
 647     exporter->options = NULL;
 648     exporter->probability = 0;
 649 }
 650
 651 static void
 652 dpif_ipfix_bridge_exporter_clear(struct dpif_ipfix_bridge_exporter *exporter)
 653 {
 654     dpif_ipfix_exporter_clear(&exporter->exporter);
 655     ofproto_ipfix_bridge_exporter_options_destroy(exporter->options);
 656     exporter->options = NULL;
 657     exporter->probability = 0;
 658 }
 659
 660 static void
 661 dpif_ipfix_bridge_exporter_destroy(struct dpif_ipfix_bridge_exporter *exporter)
 662 {
 663     dpif_ipfix_bridge_exporter_clear(exporter);
 664     dpif_ipfix_exporter_destroy(&exporter->exporter);
 665 }
 666
 667 static void
 668 dpif_ipfix_bridge_exporter_set_options(
 669     struct dpif_ipfix_bridge_exporter *exporter,
 670     const struct ofproto_ipfix_bridge_exporter_options *options)
 671 {
 672     bool options_changed;
 673
 674     if (!options || sset_is_empty(&options->targets)) {
 675         /* No point in doing any work if there are no targets. */
 676         dpif_ipfix_bridge_exporter_clear(exporter);
 677         return;
 678     }
 679
 680     options_changed = (
 681         !exporter->options
 682         || !ofproto_ipfix_bridge_exporter_options_equal(
 683             options, exporter->options));
 684
 685     /* Configure collectors if options have changed or if we're
 686      * shortchanged in collectors (which indicates that opening one or
 687      * more of the configured collectors failed, so that we should
 688      * retry). */
 689     if (options_changed
 690         || collectors_count(exporter->exporter.collectors)
 691             < sset_count(&options->targets)) {
 692         if (!dpif_ipfix_exporter_set_options(
 693                 &exporter->exporter, &options->targets,
 694                 options->cache_active_timeout, options->cache_max_flows)) {
 695             return;
 696         }
 697     }
 698
 699     /* Avoid reconfiguring if options didn't change. */
 700     if (!options_changed) {
 701         return;
 702     }
 703
 704     ofproto_ipfix_bridge_exporter_options_destroy(exporter->options);
 705     exporter->options = ofproto_ipfix_bridge_exporter_options_clone(options);
 706     exporter->probability =
 707         MAX(1, UINT32_MAX / exporter->options->sampling_rate);
 708
 709     /* Run over the cache as some entries might have expired after
 710      * changing the timeouts. */
 711     dpif_ipfix_cache_expire_now(&exporter->exporter, false);
 712 }
 713
 714 static struct dpif_ipfix_flow_exporter_map_node*
 715 dpif_ipfix_find_flow_exporter_map_node(
 716     const struct dpif_ipfix *di, const uint32_t collector_set_id)
 717     OVS_REQUIRES(mutex)
 718 {
 719     struct dpif_ipfix_flow_exporter_map_node *exporter_node;
 720
 721     HMAP_FOR_EACH_WITH_HASH (exporter_node, node,
 722                              hash_int(collector_set_id, 0),
 723                              &di->flow_exporter_map) {
 724         if (exporter_node->exporter.options->collector_set_id
 725             == collector_set_id) {
 726             return exporter_node;
 727         }
 728     }
 729
 730     return NULL;
 731 }
 732
 733 static void
 734 dpif_ipfix_flow_exporter_init(struct dpif_ipfix_flow_exporter *exporter)
 735 {
 736     dpif_ipfix_exporter_init(&exporter->exporter);
 737     exporter->options = NULL;
 738 }
 739
 740 static void
 741 dpif_ipfix_flow_exporter_clear(struct dpif_ipfix_flow_exporter *exporter)
 742 {
 743     dpif_ipfix_exporter_clear(&exporter->exporter);
 744     ofproto_ipfix_flow_exporter_options_destroy(exporter->options);
 745     exporter->options = NULL;
 746 }
 747
 748 static void
 749 dpif_ipfix_flow_exporter_destroy(struct dpif_ipfix_flow_exporter *exporter)
 750 {
 751     dpif_ipfix_flow_exporter_clear(exporter);
 752     dpif_ipfix_exporter_destroy(&exporter->exporter);
 753 }
 754
 755 static bool
 756 dpif_ipfix_flow_exporter_set_options(
 757     struct dpif_ipfix_flow_exporter *exporter,
 758     const struct ofproto_ipfix_flow_exporter_options *options)
 759 {
 760     bool options_changed;
 761
 762     if (sset_is_empty(&options->targets)) {
 763         /* No point in doing any work if there are no targets. */
 764         dpif_ipfix_flow_exporter_clear(exporter);
 765         return true;
 766     }
 767
 768     options_changed = (
 769         !exporter->options
 770         || !ofproto_ipfix_flow_exporter_options_equal(
 771             options, exporter->options));
 772
 773     /* Configure collectors if options have changed or if we're
 774      * shortchanged in collectors (which indicates that opening one or
 775      * more of the configured collectors failed, so that we should
 776      * retry). */
 777     if (options_changed
 778         || collectors_count(exporter->exporter.collectors)
 779             < sset_count(&options->targets)) {
 780         if (!dpif_ipfix_exporter_set_options(
 781                 &exporter->exporter, &options->targets,
 782                 options->cache_active_timeout, options->cache_max_flows)) {
 783             return false;
 784         }
 785     }
 786
 787     /* Avoid reconfiguring if options didn't change. */
 788     if (!options_changed) {
 789         return true;
 790     }
 791
 792     ofproto_ipfix_flow_exporter_options_destroy(exporter->options);
 793     exporter->options = ofproto_ipfix_flow_exporter_options_clone(options);
 794
 795     /* Run over the cache as some entries might have expired after
 796      * changing the timeouts. */
 797     dpif_ipfix_cache_expire_now(&exporter->exporter, false);
 798
 799     return true;
 800 }
 801
 802 void
 803 dpif_ipfix_set_options(
 804     struct dpif_ipfix *di,
 805     const struct ofproto_ipfix_bridge_exporter_options *bridge_exporter_options,
 806     const struct ofproto_ipfix_flow_exporter_options *flow_exporters_options,
 807     size_t n_flow_exporters_options) OVS_EXCLUDED(mutex)
 808 {
 809     int i;
 810     struct ofproto_ipfix_flow_exporter_options *options;
 811     struct dpif_ipfix_flow_exporter_map_node *node, *next;
 812     size_t n_broken_flow_exporters_options = 0;
 813
 814     ovs_mutex_lock(&mutex);
 815     dpif_ipfix_bridge_exporter_set_options(&di->bridge_exporter,
 816                                            bridge_exporter_options);
 817
 818     /* Add new flow exporters and update current flow exporters. */
 819     options = (struct ofproto_ipfix_flow_exporter_options *)
 820         flow_exporters_options;
 821     for (i = 0; i < n_flow_exporters_options; i++) {
 822         node = dpif_ipfix_find_flow_exporter_map_node(
 823             di, options->collector_set_id);
 824         if (!node) {
 825             node = xzalloc(sizeof *node);
 826             dpif_ipfix_flow_exporter_init(&node->exporter);
 827             hmap_insert(&di->flow_exporter_map, &node->node,
 828                         hash_int(options->collector_set_id, 0));
 829         }
 830         if (!dpif_ipfix_flow_exporter_set_options(&node->exporter, options)) {
 831             n_broken_flow_exporters_options++;
 832         }
 833         options++;
 834     }
 835
 836     ovs_assert(hmap_count(&di->flow_exporter_map) >=
 837                (n_flow_exporters_options - n_broken_flow_exporters_options));
 838
 839     /* Remove dropped flow exporters, if any needs to be removed. */
 840     if (hmap_count(&di->flow_exporter_map) > n_flow_exporters_options) {
 841         HMAP_FOR_EACH_SAFE (node, next, node, &di->flow_exporter_map) {
 842             /* This is slow but doesn't take any extra memory, and
 843              * this table is not supposed to contain many rows anyway. */
 844             options = (struct ofproto_ipfix_flow_exporter_options *)
 845                 flow_exporters_options;
 846             for (i = 0; i < n_flow_exporters_options; i++) {
 847               if (node->exporter.options->collector_set_id
 848                   == options->collector_set_id) {
 849                   break;
 850               }
 851               options++;
 852             }
 853             if (i == n_flow_exporters_options) {  // Not found.
 854                 hmap_remove(&di->flow_exporter_map, &node->node);
 855                 dpif_ipfix_flow_exporter_destroy(&node->exporter);
 856                 free(node);
 857             }
 858         }
 859     }
 860
 861     ovs_assert(hmap_count(&di->flow_exporter_map) ==
 862                (n_flow_exporters_options - n_broken_flow_exporters_options));
 863     ovs_mutex_unlock(&mutex);
 864 }
 865
 866 struct dpif_ipfix *
 867 dpif_ipfix_create(void)
 868 {
 869     struct dpif_ipfix *di;
 870     di = xzalloc(sizeof *di);
 871     dpif_ipfix_bridge_exporter_init(&di->bridge_exporter);
 872     hmap_init(&di->flow_exporter_map);
 873     hmap_init(&di->tunnel_ports);
 874     ovs_refcount_init(&di->ref_cnt);
 875     return di;
 876 }
 877
 878 struct dpif_ipfix *
 879 dpif_ipfix_ref(const struct dpif_ipfix *di_)
 880 {
 881     struct dpif_ipfix *di = CONST_CAST(struct dpif_ipfix *, di_);
 882     if (di) {
 883         ovs_refcount_ref(&di->ref_cnt);
 884     }
 885     return di;
 886 }
 887
 888 uint32_t
 889 dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *di)
 890     OVS_EXCLUDED(mutex)
 891 {
 892     uint32_t ret;
 893     ovs_mutex_lock(&mutex);
 894     ret = di->bridge_exporter.probability;
 895     ovs_mutex_unlock(&mutex);
 896     return ret;
 897 }
 898
 899 bool
 900 dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *di)
 901     OVS_EXCLUDED(mutex)
 902 {
 903     bool ret = true;
 904     ovs_mutex_lock(&mutex);
 905     if (di->bridge_exporter.options) {
 906         ret = di->bridge_exporter.options->enable_input_sampling;
 907     }
 908     ovs_mutex_unlock(&mutex);
 909     return ret;
 910 }
 911
 912 bool
 913 dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *di)
 914     OVS_EXCLUDED(mutex)
 915 {
 916     bool ret = true;
 917     ovs_mutex_lock(&mutex);
 918     if (di->bridge_exporter.options) {
 919         ret = di->bridge_exporter.options->enable_output_sampling;
 920     }
 921     ovs_mutex_unlock(&mutex);
 922     return ret;
 923 }
 924
 925 bool
 926 dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *di)
 927     OVS_EXCLUDED(mutex)
 928 {
 929     bool ret = false;
 930     ovs_mutex_lock(&mutex);
 931     if (di->bridge_exporter.options) {
 932         ret = di->bridge_exporter.options->enable_tunnel_sampling;
 933     }
 934     ovs_mutex_unlock(&mutex);
 935     return ret;
 936 }
 937
 938 static void
 939 dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
 940 {
 941     struct dpif_ipfix_flow_exporter_map_node *exp_node, *exp_next;
 942     struct dpif_ipfix_port *dip, *next;
 943
 944     dpif_ipfix_bridge_exporter_clear(&di->bridge_exporter);
 945
 946     HMAP_FOR_EACH_SAFE (exp_node, exp_next, node, &di->flow_exporter_map) {
 947         hmap_remove(&di->flow_exporter_map, &exp_node->node);
 948         dpif_ipfix_flow_exporter_destroy(&exp_node->exporter);
 949         free(exp_node);
 950     }
 951
 952     HMAP_FOR_EACH_SAFE (dip, next, hmap_node, &di->tunnel_ports) {
 953         dpif_ipfix_del_port(di, dip);
 954     }
 955 }
 956
 957 void
 958 dpif_ipfix_unref(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
 959 {
 960     if (di && ovs_refcount_unref_relaxed(&di->ref_cnt) == 1) {
 961         ovs_mutex_lock(&mutex);
 962         dpif_ipfix_clear(di);
 963         dpif_ipfix_bridge_exporter_destroy(&di->bridge_exporter);
 964         hmap_destroy(&di->flow_exporter_map);
 965         hmap_destroy(&di->tunnel_ports);
 966         free(di);
 967         ovs_mutex_unlock(&mutex);
 968     }
 969 }
 970
 971 static void
 972 ipfix_init_header(uint32_t export_time_sec, uint32_t seq_number,
 973                   uint32_t obs_domain_id, struct ofpbuf *msg)
 974 {
 975     struct ipfix_header *hdr;
 976
 977     hdr = ofpbuf_put_zeros(msg, sizeof *hdr);
 978     hdr->version = htons(IPFIX_VERSION);
 979     hdr->length = htons(sizeof *hdr);  /* Updated in ipfix_send_msg. */
 980     hdr->export_time = htonl(export_time_sec);
 981     hdr->seq_number = htonl(seq_number);
 982     hdr->obs_domain_id = htonl(obs_domain_id);
 983 }
 984
 985 static void
 986 ipfix_send_msg(const struct collectors *collectors, struct ofpbuf *msg)
 987 {
 988     struct ipfix_header *hdr;
 989
 990     /* Adjust the length in the header. */
 991     hdr = ofpbuf_data(msg);
 992     hdr->length = htons(ofpbuf_size(msg));
 993
 994     collectors_send(collectors, ofpbuf_data(msg), ofpbuf_size(msg));
 995     ofpbuf_set_size(msg, 0);
 996 }
 997
 998 static uint16_t
 999 ipfix_get_template_id(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
1000                       enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel)
1001 {
1002     uint16_t template_id;
1003     template_id = l2;
1004     template_id = template_id * NUM_IPFIX_PROTO_L3 + l3;
1005     template_id = template_id * NUM_IPFIX_PROTO_L4 + l4;
1006     template_id = template_id * NUM_IPFIX_PROTO_TUNNEL + tunnel;
1007     return IPFIX_TEMPLATE_ID_MIN + template_id;
1008 }
1009
1010 static void
1011 ipfix_define_template_entity(enum ipfix_entity_id id,
1012                              enum ipfix_entity_size size,
1013                              enum ipfix_entity_enterprise enterprise,
1014                              struct ofpbuf *msg)
1015 {
1016     struct ipfix_template_field_specifier *field;
1017     size_t field_size;
1018
1019     if (enterprise) {
1020         field_size = sizeof *field;
1021     } else {
1022         /* No enterprise number */
1023         field_size = sizeof *field - sizeof(ovs_be32);
1024     }
1025     field = ofpbuf_put_zeros(msg, field_size);
1026     field->element_id = htons(id);
1027     if (size) {
1028         field->field_length = htons(size);
1029     } else {
1030         /* RFC 5101, Section 7. Variable-Length Information Element */
1031         field->field_length = OVS_BE16_MAX;
1032     }
1033     if (enterprise) {
1034         field->enterprise = htonl(enterprise);
1035     }
1036
1037 }
1038
1039 static uint16_t
1040 ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
1041                              enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel,
1042                              struct ofpbuf *msg)
1043 {
1044     uint16_t count = 0;
1045
1046 #define DEF(ID) \
1047     { \
1048         ipfix_define_template_entity(IPFIX_ENTITY_ID_##ID, \
1049                                      IPFIX_ENTITY_SIZE_##ID, \
1050                                      IPFIX_ENTITY_ENTERPRISE_##ID, msg); \
1051         count++; \
1052     }
1053
1054     /* 1. Flow key. */
1055
1056     DEF(OBSERVATION_POINT_ID);
1057     DEF(FLOW_DIRECTION);
1058
1059     /* Common Ethernet entities. */
1060     DEF(SOURCE_MAC_ADDRESS);
1061     DEF(DESTINATION_MAC_ADDRESS);
1062     DEF(ETHERNET_TYPE);
1063     DEF(ETHERNET_HEADER_LENGTH);
1064
1065     if (l2 == IPFIX_PROTO_L2_VLAN) {
1066         DEF(VLAN_ID);
1067         DEF(DOT1Q_VLAN_ID);
1068         DEF(DOT1Q_PRIORITY);
1069     }
1070
1071     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1072         DEF(IP_VERSION);
1073         DEF(IP_TTL);
1074         DEF(PROTOCOL_IDENTIFIER);
1075         DEF(IP_DIFF_SERV_CODE_POINT);
1076         DEF(IP_PRECEDENCE);
1077         DEF(IP_CLASS_OF_SERVICE);
1078
1079         if (l3 == IPFIX_PROTO_L3_IPV4) {
1080             DEF(SOURCE_IPV4_ADDRESS);
1081             DEF(DESTINATION_IPV4_ADDRESS);
1082             if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1083                 DEF(SOURCE_TRANSPORT_PORT);
1084                 DEF(DESTINATION_TRANSPORT_PORT);
1085             } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1086                 DEF(ICMP_TYPE_IPV4);
1087                 DEF(ICMP_CODE_IPV4);
1088             }
1089         } else {  /* l3 == IPFIX_PROTO_L3_IPV6 */
1090             DEF(SOURCE_IPV6_ADDRESS);
1091             DEF(DESTINATION_IPV6_ADDRESS);
1092             DEF(FLOW_LABEL_IPV6);
1093             if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1094                 DEF(SOURCE_TRANSPORT_PORT);
1095                 DEF(DESTINATION_TRANSPORT_PORT);
1096             } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1097                 DEF(ICMP_TYPE_IPV6);
1098                 DEF(ICMP_CODE_IPV6);
1099             }
1100         }
1101     }
1102
1103     if (tunnel != IPFIX_PROTO_NOT_TUNNELED) {
1104         DEF(TUNNEL_SOURCE_IPV4_ADDRESS);
1105         DEF(TUNNEL_DESTINATION_IPV4_ADDRESS);
1106         DEF(TUNNEL_PROTOCOL_IDENTIFIER);
1107         DEF(TUNNEL_SOURCE_TRANSPORT_PORT);
1108         DEF(TUNNEL_DESTINATION_TRANSPORT_PORT);
1109         DEF(TUNNEL_TYPE);
1110         DEF(TUNNEL_KEY);
1111     }
1112
1113     /* 2. Flow aggregated data. */
1114
1115     DEF(FLOW_START_DELTA_MICROSECONDS);
1116     DEF(FLOW_END_DELTA_MICROSECONDS);
1117     DEF(PACKET_DELTA_COUNT);
1118     DEF(LAYER2_OCTET_DELTA_COUNT);
1119     DEF(FLOW_END_REASON);
1120
1121     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1122         DEF(OCTET_DELTA_COUNT);
1123         DEF(OCTET_DELTA_SUM_OF_SQUARES);
1124         DEF(MINIMUM_IP_TOTAL_LENGTH);
1125         DEF(MAXIMUM_IP_TOTAL_LENGTH);
1126     }
1127
1128
1129 #undef DEF
1130
1131     return count;
1132 }
1133
1134 static void
1135 ipfix_init_template_msg(void *msg_stub, uint32_t export_time_sec,
1136                         uint32_t seq_number, uint32_t obs_domain_id,
1137                         struct ofpbuf *msg, size_t *set_hdr_offset)
1138 {
1139     struct ipfix_set_header *set_hdr;
1140
1141     ofpbuf_use_stub(msg, msg_stub, sizeof msg_stub);
1142
1143     ipfix_init_header(export_time_sec, seq_number, obs_domain_id, msg);
1144     *set_hdr_offset = ofpbuf_size(msg);
1145
1146     /* Add a Template Set. */
1147     set_hdr = ofpbuf_put_zeros(msg, sizeof *set_hdr);
1148     set_hdr->set_id = htons(IPFIX_SET_ID_TEMPLATE);
1149 }
1150
1151 static void
1152 ipfix_send_template_msg(const struct collectors *collectors,
1153                         struct ofpbuf *msg, size_t set_hdr_offset)
1154 {
1155     struct ipfix_set_header *set_hdr;
1156
1157     /* Send template message. */
1158     set_hdr = (struct ipfix_set_header*)
1159               ((uint8_t*)ofpbuf_data(msg) + set_hdr_offset);
1160     set_hdr->length = htons(ofpbuf_size(msg) - set_hdr_offset);
1161
1162     ipfix_send_msg(collectors, msg);
1163
1164     ofpbuf_uninit(msg);
1165 }
1166
1167 static void
1168 ipfix_send_template_msgs(struct dpif_ipfix_exporter *exporter,
1169                          uint32_t export_time_sec, uint32_t obs_domain_id)
1170 {
1171     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
1172     struct ofpbuf msg;
1173     size_t set_hdr_offset, tmpl_hdr_offset;
1174     struct ipfix_template_record_header *tmpl_hdr;
1175     uint16_t field_count;
1176     enum ipfix_proto_l2 l2;
1177     enum ipfix_proto_l3 l3;
1178     enum ipfix_proto_l4 l4;
1179     enum ipfix_proto_tunnel tunnel;
1180
1181     ipfix_init_template_msg(msg_stub, export_time_sec, exporter->seq_number,
1182                             obs_domain_id, &msg, &set_hdr_offset);
1183     /* Define one template for each possible combination of
1184      * protocols. */
1185     for (l2 = 0; l2 < NUM_IPFIX_PROTO_L2; l2++) {
1186         for (l3 = 0; l3 < NUM_IPFIX_PROTO_L3; l3++) {
1187             for (l4 = 0; l4 < NUM_IPFIX_PROTO_L4; l4++) {
1188                 if (l3 == IPFIX_PROTO_L3_UNKNOWN &&
1189                     l4 != IPFIX_PROTO_L4_UNKNOWN) {
1190                     continue;
1191                 }
1192                 for (tunnel = 0; tunnel < NUM_IPFIX_PROTO_TUNNEL; tunnel++) {
1193                     /* When the size of the template packet reaches
1194                      * MAX_MESSAGE_LEN(1024), send it out.
1195                      * And then reinitialize the msg to construct a new
1196                      * packet for the following templates.
1197                      */
1198                     if (ofpbuf_size(&msg) >= MAX_MESSAGE_LEN) {
1199                         /* Send template message. */
1200                         ipfix_send_template_msg(exporter->collectors,
1201                                                 &msg, set_hdr_offset);
1202
1203                         /* Reinitialize the template msg. */
1204                         ipfix_init_template_msg(msg_stub, export_time_sec,
1205                                                 exporter->seq_number,
1206                                                 obs_domain_id, &msg,
1207                                                 &set_hdr_offset);
1208                     }
1209
1210                     tmpl_hdr_offset = ofpbuf_size(&msg);
1211                     tmpl_hdr = ofpbuf_put_zeros(&msg, sizeof *tmpl_hdr);
1212                     tmpl_hdr->template_id = htons(
1213                         ipfix_get_template_id(l2, l3, l4, tunnel));
1214                     field_count =
1215                         ipfix_define_template_fields(l2, l3, l4, tunnel, &msg);
1216                     tmpl_hdr = (struct ipfix_template_record_header*)
1217                         ((uint8_t*)ofpbuf_data(&msg) + tmpl_hdr_offset);
1218                     tmpl_hdr->field_count = htons(field_count);
1219                 }
1220             }
1221         }
1222     }
1223
1224     /* Send template message. */
1225     ipfix_send_template_msg(exporter->collectors, &msg, set_hdr_offset);
1226
1227     /* XXX: Add Options Template Sets, at least to define a Flow Keys
1228      * Option Template. */
1229
1230 }
1231
1232 static inline uint32_t
1233 ipfix_hash_flow_key(const struct ipfix_flow_key *flow_key, uint32_t basis)
1234 {
1235     uint32_t hash;
1236     hash = hash_int(flow_key->obs_domain_id, basis);
1237     hash = hash_int(flow_key->template_id, hash);
1238     hash = hash_bytes(flow_key->flow_key_msg_part,
1239                       flow_key->flow_key_msg_part_size, hash);
1240     return hash;
1241 }
1242
1243 static bool
1244 ipfix_flow_key_equal(const struct ipfix_flow_key *a,
1245                      const struct ipfix_flow_key *b)
1246 {
1247     /* The template ID determines the flow key size, so not need to
1248      * compare it. */
1249     return (a->obs_domain_id == b->obs_domain_id
1250             && a->template_id == b->template_id
1251             && memcmp(a->flow_key_msg_part, b->flow_key_msg_part,
1252                       a->flow_key_msg_part_size) == 0);
1253 }
1254
1255 static struct ipfix_flow_cache_entry*
1256 ipfix_cache_find_entry(const struct dpif_ipfix_exporter *exporter,
1257                        const struct ipfix_flow_key *flow_key)
1258 {
1259     struct ipfix_flow_cache_entry *entry;
1260
1261     HMAP_FOR_EACH_WITH_HASH (entry, flow_key_map_node,
1262                              ipfix_hash_flow_key(flow_key, 0),
1263                              &exporter->cache_flow_key_map) {
1264         if (ipfix_flow_key_equal(&entry->flow_key, flow_key)) {
1265             return entry;
1266         }
1267     }
1268
1269     return NULL;
1270 }
1271
1272 static bool
1273 ipfix_cache_next_timeout_msec(const struct dpif_ipfix_exporter *exporter,
1274                               long long int *next_timeout_msec)
1275 {
1276     struct ipfix_flow_cache_entry *entry;
1277
1278     LIST_FOR_EACH (entry, cache_flow_start_timestamp_list_node,
1279                    &exporter->cache_flow_start_timestamp_list) {
1280         *next_timeout_msec = entry->flow_start_timestamp_usec / 1000LL
1281             + 1000LL * exporter->cache_active_timeout;
1282         return true;
1283     }
1284
1285     return false;
1286 }
1287
1288 static void
1289 ipfix_cache_aggregate_entries(struct ipfix_flow_cache_entry *from_entry,
1290                               struct ipfix_flow_cache_entry *to_entry)
1291 {
1292     uint64_t *to_start, *to_end, *from_start, *from_end;
1293     uint16_t *to_min_len, *to_max_len, *from_min_len, *from_max_len;
1294
1295     to_start = &to_entry->flow_start_timestamp_usec;
1296     to_end = &to_entry->flow_end_timestamp_usec;
1297     from_start = &from_entry->flow_start_timestamp_usec;
1298     from_end = &from_entry->flow_end_timestamp_usec;
1299
1300     if (*to_start > *from_start) {
1301         *to_start = *from_start;
1302     }
1303     if (*to_end < *from_end) {
1304         *to_end = *from_end;
1305     }
1306
1307     to_entry->packet_delta_count += from_entry->packet_delta_count;
1308     to_entry->layer2_octet_delta_count += from_entry->layer2_octet_delta_count;
1309
1310     to_entry->octet_delta_count += from_entry->octet_delta_count;
1311     to_entry->octet_delta_sum_of_squares +=
1312         from_entry->octet_delta_sum_of_squares;
1313
1314     to_min_len = &to_entry->minimum_ip_total_length;
1315     to_max_len = &to_entry->maximum_ip_total_length;
1316     from_min_len = &from_entry->minimum_ip_total_length;
1317     from_max_len = &from_entry->maximum_ip_total_length;
1318
1319     if (!*to_min_len || (*from_min_len && *to_min_len > *from_min_len)) {
1320         *to_min_len = *from_min_len;
1321     }
1322     if (*to_max_len < *from_max_len) {
1323         *to_max_len = *from_max_len;
1324     }
1325 }
1326
1327 /* Add an entry into a flow cache.  The entry is either aggregated into
1328  * an existing entry with the same flow key and free()d, or it is
1329  * inserted into the cache. */
1330 static void
1331 ipfix_cache_update(struct dpif_ipfix_exporter *exporter,
1332                    struct ipfix_flow_cache_entry *entry)
1333 {
1334     struct ipfix_flow_cache_entry *old_entry;
1335
1336     old_entry = ipfix_cache_find_entry(exporter, &entry->flow_key);
1337
1338     if (old_entry == NULL) {
1339         hmap_insert(&exporter->cache_flow_key_map, &entry->flow_key_map_node,
1340                     ipfix_hash_flow_key(&entry->flow_key, 0));
1341
1342         /* As the latest entry added into the cache, it should
1343          * logically have the highest flow_start_timestamp_usec, so
1344          * append it at the tail. */
1345         list_push_back(&exporter->cache_flow_start_timestamp_list,
1346                        &entry->cache_flow_start_timestamp_list_node);
1347
1348         /* Enforce exporter->cache_max_flows limit. */
1349         if (hmap_count(&exporter->cache_flow_key_map)
1350             > exporter->cache_max_flows) {
1351             dpif_ipfix_cache_expire_now(exporter, false);
1352         }
1353     } else {
1354         ipfix_cache_aggregate_entries(entry, old_entry);
1355         free(entry);
1356     }
1357 }
1358
1359 static void
1360 ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
1361                        const struct ofpbuf *packet, const struct flow *flow,
1362                        uint64_t packet_delta_count, uint32_t obs_domain_id,
1363                        uint32_t obs_point_id, odp_port_t output_odp_port,
1364                        const struct dpif_ipfix_port *tunnel_port,
1365                        const struct flow_tnl *tunnel_key)
1366 {
1367     struct ipfix_flow_key *flow_key;
1368     struct ofpbuf msg;
1369     enum ipfix_proto_l2 l2;
1370     enum ipfix_proto_l3 l3;
1371     enum ipfix_proto_l4 l4;
1372     enum ipfix_proto_tunnel tunnel = IPFIX_PROTO_NOT_TUNNELED;
1373     uint8_t ethernet_header_length;
1374     uint16_t ethernet_total_length;
1375
1376     flow_key = &entry->flow_key;
1377     ofpbuf_use_stack(&msg, flow_key->flow_key_msg_part,
1378                      sizeof flow_key->flow_key_msg_part);
1379
1380     /* Choose the right template ID matching the protocols in the
1381      * sampled packet. */
1382     l2 = (flow->vlan_tci == 0) ? IPFIX_PROTO_L2_ETH : IPFIX_PROTO_L2_VLAN;
1383
1384     switch(ntohs(flow->dl_type)) {
1385     case ETH_TYPE_IP:
1386         l3 = IPFIX_PROTO_L3_IPV4;
1387         switch(flow->nw_proto) {
1388         case IPPROTO_TCP:
1389         case IPPROTO_UDP:
1390         case IPPROTO_SCTP:
1391             l4 = IPFIX_PROTO_L4_TCP_UDP_SCTP;
1392             break;
1393         case IPPROTO_ICMP:
1394             l4 = IPFIX_PROTO_L4_ICMP;
1395             break;
1396         default:
1397             l4 = IPFIX_PROTO_L4_UNKNOWN;
1398         }
1399         break;
1400     case ETH_TYPE_IPV6:
1401         l3 = IPFIX_PROTO_L3_IPV6;
1402         switch(flow->nw_proto) {
1403         case IPPROTO_TCP:
1404         case IPPROTO_UDP:
1405         case IPPROTO_SCTP:
1406             l4 = IPFIX_PROTO_L4_TCP_UDP_SCTP;
1407             break;
1408         case IPPROTO_ICMPV6:
1409             l4 = IPFIX_PROTO_L4_ICMP;
1410             break;
1411         default:
1412             l4 = IPFIX_PROTO_L4_UNKNOWN;
1413         }
1414         break;
1415     default:
1416         l3 = IPFIX_PROTO_L3_UNKNOWN;
1417         l4 = IPFIX_PROTO_L4_UNKNOWN;
1418     }
1419
1420     if (tunnel_port && tunnel_key) {
1421        tunnel = IPFIX_PROTO_TUNNELED;
1422     }
1423
1424     flow_key->obs_domain_id = obs_domain_id;
1425     flow_key->template_id = ipfix_get_template_id(l2, l3, l4, tunnel);
1426
1427     /* The fields defined in the ipfix_data_record_* structs and sent
1428      * below must match exactly the templates defined in
1429      * ipfix_define_template_fields. */
1430
1431     ethernet_header_length = (l2 == IPFIX_PROTO_L2_VLAN)
1432         ? VLAN_ETH_HEADER_LEN : ETH_HEADER_LEN;
1433     ethernet_total_length = ofpbuf_size(packet);
1434
1435     /* Common Ethernet entities. */
1436     {
1437         struct ipfix_data_record_flow_key_common *data_common;
1438
1439         data_common = ofpbuf_put_zeros(&msg, sizeof *data_common);
1440         data_common->observation_point_id = htonl(obs_point_id);
1441         data_common->flow_direction =
1442             (output_odp_port == ODPP_NONE) ? INGRESS_FLOW : EGRESS_FLOW;
1443         memcpy(data_common->source_mac_address, flow->dl_src,
1444                sizeof flow->dl_src);
1445         memcpy(data_common->destination_mac_address, flow->dl_dst,
1446                sizeof flow->dl_dst);
1447         data_common->ethernet_type = flow->dl_type;
1448         data_common->ethernet_header_length = ethernet_header_length;
1449     }
1450
1451     if (l2 == IPFIX_PROTO_L2_VLAN) {
1452         struct ipfix_data_record_flow_key_vlan *data_vlan;
1453         uint16_t vlan_id = vlan_tci_to_vid(flow->vlan_tci);
1454         uint8_t priority = vlan_tci_to_pcp(flow->vlan_tci);
1455
1456         data_vlan = ofpbuf_put_zeros(&msg, sizeof *data_vlan);
1457         data_vlan->vlan_id = htons(vlan_id);
1458         data_vlan->dot1q_vlan_id = htons(vlan_id);
1459         data_vlan->dot1q_priority = priority;
1460     }
1461
1462     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1463         struct ipfix_data_record_flow_key_ip *data_ip;
1464
1465         data_ip = ofpbuf_put_zeros(&msg, sizeof *data_ip);
1466         data_ip->ip_version = (l3 == IPFIX_PROTO_L3_IPV4) ? 4 : 6;
1467         data_ip->ip_ttl = flow->nw_ttl;
1468         data_ip->protocol_identifier = flow->nw_proto;
1469         data_ip->ip_diff_serv_code_point = flow->nw_tos >> 2;
1470         data_ip->ip_precedence = flow->nw_tos >> 5;
1471         data_ip->ip_class_of_service = flow->nw_tos;
1472
1473         if (l3 == IPFIX_PROTO_L3_IPV4) {
1474             struct ipfix_data_record_flow_key_ipv4 *data_ipv4;
1475
1476             data_ipv4 = ofpbuf_put_zeros(&msg, sizeof *data_ipv4);
1477             data_ipv4->source_ipv4_address = flow->nw_src;
1478             data_ipv4->destination_ipv4_address = flow->nw_dst;
1479         } else {  /* l3 == IPFIX_PROTO_L3_IPV6 */
1480             struct ipfix_data_record_flow_key_ipv6 *data_ipv6;
1481
1482             data_ipv6 = ofpbuf_put_zeros(&msg, sizeof *data_ipv6);
1483             memcpy(data_ipv6->source_ipv6_address, &flow->ipv6_src,
1484                    sizeof flow->ipv6_src);
1485             memcpy(data_ipv6->destination_ipv6_address, &flow->ipv6_dst,
1486                    sizeof flow->ipv6_dst);
1487             data_ipv6->flow_label_ipv6 = flow->ipv6_label;
1488         }
1489     }
1490
1491     if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1492         struct ipfix_data_record_flow_key_transport *data_transport;
1493
1494         data_transport = ofpbuf_put_zeros(&msg, sizeof *data_transport);
1495         data_transport->source_transport_port = flow->tp_src;
1496         data_transport->destination_transport_port = flow->tp_dst;
1497     } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1498         struct ipfix_data_record_flow_key_icmp *data_icmp;
1499
1500         data_icmp = ofpbuf_put_zeros(&msg, sizeof *data_icmp);
1501         data_icmp->icmp_type = ntohs(flow->tp_src) & 0xff;
1502         data_icmp->icmp_code = ntohs(flow->tp_dst) & 0xff;
1503     }
1504
1505     if (tunnel == IPFIX_PROTO_TUNNELED) {
1506         struct ipfix_data_record_flow_key_tunnel *data_tunnel;
1507         const uint8_t *tun_id;
1508
1509         data_tunnel = ofpbuf_put_zeros(&msg, sizeof *data_tunnel +
1510                                              tunnel_port->tunnel_key_length);
1511         data_tunnel->tunnel_source_ipv4_address = tunnel_key->ip_src;
1512         data_tunnel->tunnel_destination_ipv4_address = tunnel_key->ip_dst;
1513         /* The tunnel_protocol_identifier is from tunnel_proto array, which
1514          * contains protocol_identifiers of each tunnel type.
1515          * For the tunnel type on the top of IPSec, which uses the protocol
1516          * identifier of the upper tunnel type is used, the tcp_src and tcp_dst
1517          * are decided based on the protocol identifiers.
1518          * E.g:
1519          * The protocol identifier of DPIF_IPFIX_TUNNEL_IPSEC_GRE is IPPROTO_GRE,
1520          * and both tp_src and tp_dst are zero.
1521          */
1522         data_tunnel->tunnel_protocol_identifier =
1523             tunnel_protocol[tunnel_port->tunnel_type];
1524         data_tunnel->tunnel_source_transport_port = tunnel_key->tp_src;
1525         data_tunnel->tunnel_destination_transport_port = tunnel_key->tp_dst;
1526         data_tunnel->tunnel_type = tunnel_port->tunnel_type;
1527         data_tunnel->tunnel_key_length = tunnel_port->tunnel_key_length;
1528         /* tun_id is in network order, and tunnel key is in low bits. */
1529         tun_id = (const uint8_t *) &tunnel_key->tun_id;
1530         memcpy(data_tunnel->tunnel_key,
1531                &tun_id[8 - tunnel_port->tunnel_key_length],
1532                tunnel_port->tunnel_key_length);
1533     }
1534
1535     flow_key->flow_key_msg_part_size = ofpbuf_size(&msg);
1536
1537     {
1538         struct timeval now;
1539         uint64_t layer2_octet_delta_count;
1540
1541         /* Calculate the total matched octet count by considering as
1542          * an approximation that all matched packets have the same
1543          * length. */
1544         layer2_octet_delta_count = packet_delta_count * ethernet_total_length;
1545
1546         xgettimeofday(&now);
1547         entry->flow_end_timestamp_usec = now.tv_usec + 1000000LL * now.tv_sec;
1548         entry->flow_start_timestamp_usec = entry->flow_end_timestamp_usec;
1549         entry->packet_delta_count = packet_delta_count;
1550         entry->layer2_octet_delta_count = layer2_octet_delta_count;
1551     }
1552
1553     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1554         uint16_t ip_total_length =
1555             ethernet_total_length - ethernet_header_length;
1556         uint64_t octet_delta_count;
1557
1558         /* Calculate the total matched octet count by considering as
1559          * an approximation that all matched packets have the same
1560          * length. */
1561         octet_delta_count = packet_delta_count * ip_total_length;
1562
1563         entry->octet_delta_count = octet_delta_count;
1564         entry->octet_delta_sum_of_squares = octet_delta_count * ip_total_length;
1565         entry->minimum_ip_total_length = ip_total_length;
1566         entry->maximum_ip_total_length = ip_total_length;
1567     } else {
1568         entry->octet_delta_sum_of_squares = 0;
1569         entry->minimum_ip_total_length = 0;
1570         entry->maximum_ip_total_length = 0;
1571     }
1572 }
1573
1574 /* Send each single data record in its own data set, to simplify the
1575  * implementation by avoiding having to group record by template ID
1576  * before sending. */
1577 static void
1578 ipfix_put_data_set(uint32_t export_time_sec,
1579                    struct ipfix_flow_cache_entry *entry,
1580                    enum ipfix_flow_end_reason flow_end_reason,
1581                    struct ofpbuf *msg)
1582 {
1583     size_t set_hdr_offset;
1584     struct ipfix_set_header *set_hdr;
1585
1586     set_hdr_offset = ofpbuf_size(msg);
1587
1588     /* Put a Data Set. */
1589     set_hdr = ofpbuf_put_zeros(msg, sizeof *set_hdr);
1590     set_hdr->set_id = htons(entry->flow_key.template_id);
1591
1592     /* Copy the flow key part of the data record. */
1593
1594     ofpbuf_put(msg, entry->flow_key.flow_key_msg_part,
1595                entry->flow_key.flow_key_msg_part_size);
1596
1597     /* Put the non-key part of the data record. */
1598
1599     {
1600         struct ipfix_data_record_aggregated_common *data_aggregated_common;
1601         uint64_t export_time_usec, flow_start_delta_usec, flow_end_delta_usec;
1602
1603         /* Calculate the negative deltas relative to the export time
1604          * in seconds sent in the header, not the exact export
1605          * time. */
1606         export_time_usec = 1000000LL * export_time_sec;
1607         flow_start_delta_usec = export_time_usec
1608             - entry->flow_start_timestamp_usec;
1609         flow_end_delta_usec = export_time_usec
1610             - entry->flow_end_timestamp_usec;
1611
1612         data_aggregated_common = ofpbuf_put_zeros(
1613             msg, sizeof *data_aggregated_common);
1614         data_aggregated_common->flow_start_delta_microseconds = htonl(
1615             flow_start_delta_usec);
1616         data_aggregated_common->flow_end_delta_microseconds = htonl(
1617             flow_end_delta_usec);
1618         data_aggregated_common->packet_delta_count = htonll(
1619             entry->packet_delta_count);
1620         data_aggregated_common->layer2_octet_delta_count = htonll(
1621             entry->layer2_octet_delta_count);
1622         data_aggregated_common->flow_end_reason = flow_end_reason;
1623     }
1624
1625     if (entry->octet_delta_sum_of_squares) {  /* IP packet. */
1626         struct ipfix_data_record_aggregated_ip *data_aggregated_ip;
1627
1628         data_aggregated_ip = ofpbuf_put_zeros(
1629             msg, sizeof *data_aggregated_ip);
1630         data_aggregated_ip->octet_delta_count = htonll(
1631             entry->octet_delta_count);
1632         data_aggregated_ip->octet_delta_sum_of_squares = htonll(
1633             entry->octet_delta_sum_of_squares);
1634         data_aggregated_ip->minimum_ip_total_length = htonll(
1635             entry->minimum_ip_total_length);
1636         data_aggregated_ip->maximum_ip_total_length = htonll(
1637             entry->maximum_ip_total_length);
1638     }
1639
1640     set_hdr = (struct ipfix_set_header*)((uint8_t*)ofpbuf_data(msg) + set_hdr_offset);
1641     set_hdr->length = htons(ofpbuf_size(msg) - set_hdr_offset);
1642 }
1643
1644 /* Send an IPFIX message with a single data record. */
1645 static void
1646 ipfix_send_data_msg(struct dpif_ipfix_exporter *exporter,
1647                     uint32_t export_time_sec,
1648                     struct ipfix_flow_cache_entry *entry,
1649                     enum ipfix_flow_end_reason flow_end_reason)
1650 {
1651     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
1652     struct ofpbuf msg;
1653     ofpbuf_use_stub(&msg, msg_stub, sizeof msg_stub);
1654
1655     ipfix_init_header(export_time_sec, exporter->seq_number++,
1656                       entry->flow_key.obs_domain_id, &msg);
1657     ipfix_put_data_set(export_time_sec, entry, flow_end_reason, &msg);
1658     ipfix_send_msg(exporter->collectors, &msg);
1659
1660     ofpbuf_uninit(&msg);
1661 }
1662
1663 static void
1664 dpif_ipfix_sample(struct dpif_ipfix_exporter *exporter,
1665                   const struct ofpbuf *packet, const struct flow *flow,
1666                   uint64_t packet_delta_count, uint32_t obs_domain_id,
1667                   uint32_t obs_point_id, odp_port_t output_odp_port,
1668                   const struct dpif_ipfix_port *tunnel_port,
1669                   const struct flow_tnl *tunnel_key)
1670 {
1671     struct ipfix_flow_cache_entry *entry;
1672
1673     /* Create a flow cache entry from the sample. */
1674     entry = xmalloc(sizeof *entry);
1675     ipfix_cache_entry_init(entry, packet, flow, packet_delta_count,
1676                            obs_domain_id, obs_point_id,
1677                            output_odp_port, tunnel_port, tunnel_key);
1678     ipfix_cache_update(exporter, entry);
1679 }
1680
1681 void
1682 dpif_ipfix_bridge_sample(struct dpif_ipfix *di, const struct ofpbuf *packet,
1683                          const struct flow *flow,
1684                          odp_port_t input_odp_port, odp_port_t output_odp_port,
1685                          const struct flow_tnl *output_tunnel_key)
1686     OVS_EXCLUDED(mutex)
1687 {
1688     uint64_t packet_delta_count;
1689     const struct flow_tnl *tunnel_key = NULL;
1690     struct dpif_ipfix_port * tunnel_port = NULL;
1691
1692     ovs_mutex_lock(&mutex);
1693     /* Use the sampling probability as an approximation of the number
1694      * of matched packets. */
1695     packet_delta_count = UINT32_MAX / di->bridge_exporter.probability;
1696     if (di->bridge_exporter.options->enable_tunnel_sampling) {
1697         if (output_odp_port == ODPP_NONE && flow->tunnel.ip_dst) {
1698             /* Input tunnel. */
1699             tunnel_key = &flow->tunnel;
1700             tunnel_port = dpif_ipfix_find_port(di, input_odp_port);
1701         }
1702         if (output_odp_port != ODPP_NONE && output_tunnel_key) {
1703             /* Output tunnel, output_tunnel_key must be valid. */
1704             tunnel_key = output_tunnel_key;
1705             tunnel_port = dpif_ipfix_find_port(di, output_odp_port);
1706         }
1707     }
1708     dpif_ipfix_sample(&di->bridge_exporter.exporter, packet, flow,
1709                       packet_delta_count,
1710                       di->bridge_exporter.options->obs_domain_id,
1711                       di->bridge_exporter.options->obs_point_id,
1712                       output_odp_port, tunnel_port, tunnel_key);
1713     ovs_mutex_unlock(&mutex);
1714 }
1715
1716 void
1717 dpif_ipfix_flow_sample(struct dpif_ipfix *di, const struct ofpbuf *packet,
1718                        const struct flow *flow, uint32_t collector_set_id,
1719                        uint16_t probability, uint32_t obs_domain_id,
1720                        uint32_t obs_point_id) OVS_EXCLUDED(mutex)
1721 {
1722     struct dpif_ipfix_flow_exporter_map_node *node;
1723     /* Use the sampling probability as an approximation of the number
1724      * of matched packets. */
1725     uint64_t packet_delta_count = USHRT_MAX / probability;
1726
1727     ovs_mutex_lock(&mutex);
1728     node = dpif_ipfix_find_flow_exporter_map_node(di, collector_set_id);
1729     if (node) {
1730         dpif_ipfix_sample(&node->exporter.exporter, packet, flow,
1731                           packet_delta_count, obs_domain_id, obs_point_id,
1732                           ODPP_NONE, NULL, NULL);
1733     }
1734     ovs_mutex_unlock(&mutex);
1735 }
1736
1737 static void
1738 dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *exporter,
1739                         bool forced_end, const uint64_t export_time_usec,
1740                         const uint32_t export_time_sec)
1741 {
1742     struct ipfix_flow_cache_entry *entry, *next_entry;
1743     uint64_t max_flow_start_timestamp_usec;
1744     bool template_msg_sent = false;
1745     enum ipfix_flow_end_reason flow_end_reason;
1746
1747     if (list_is_empty(&exporter->cache_flow_start_timestamp_list)) {
1748         return;
1749     }
1750
1751     max_flow_start_timestamp_usec = export_time_usec -
1752         1000000LL * exporter->cache_active_timeout;
1753
1754     LIST_FOR_EACH_SAFE (entry, next_entry, cache_flow_start_timestamp_list_node,
1755                         &exporter->cache_flow_start_timestamp_list) {
1756         if (forced_end) {
1757             flow_end_reason = FORCED_END;
1758         } else if (entry->flow_start_timestamp_usec
1759                    <= max_flow_start_timestamp_usec) {
1760             flow_end_reason = ACTIVE_TIMEOUT;
1761         } else if (hmap_count(&exporter->cache_flow_key_map)
1762                    > exporter->cache_max_flows) {
1763             /* Enforce exporter->cache_max_flows. */
1764             flow_end_reason = LACK_OF_RESOURCES;
1765         } else {
1766             /* Remaining flows haven't expired yet. */
1767             break;
1768         }
1769
1770         list_remove(&entry->cache_flow_start_timestamp_list_node);
1771         hmap_remove(&exporter->cache_flow_key_map,
1772                     &entry->flow_key_map_node);
1773
1774         if (!template_msg_sent
1775             && (exporter->last_template_set_time + IPFIX_TEMPLATE_INTERVAL)
1776                 <= export_time_sec) {
1777             ipfix_send_template_msgs(exporter, export_time_sec,
1778                                      entry->flow_key.obs_domain_id);
1779             exporter->last_template_set_time = export_time_sec;
1780             template_msg_sent = true;
1781         }
1782
1783         /* XXX: Group multiple data records for the same obs domain id
1784          * into the same message. */
1785         ipfix_send_data_msg(exporter, export_time_sec, entry, flow_end_reason);
1786         free(entry);
1787     }
1788 }
1789
1790 static void
1791 get_export_time_now(uint64_t *export_time_usec, uint32_t *export_time_sec)
1792 {
1793     struct timeval export_time;
1794     xgettimeofday(&export_time);
1795
1796     *export_time_usec = export_time.tv_usec + 1000000LL * export_time.tv_sec;
1797
1798     /* The IPFIX start and end deltas are negative deltas relative to
1799      * the export time, so set the export time 1 second off to
1800      * calculate those deltas. */
1801     if (export_time.tv_usec == 0) {
1802         *export_time_sec = export_time.tv_sec;
1803     } else {
1804         *export_time_sec = export_time.tv_sec + 1;
1805     }
1806 }
1807
1808 static void
1809 dpif_ipfix_cache_expire_now(struct dpif_ipfix_exporter *exporter,
1810                             bool forced_end)
1811 {
1812     uint64_t export_time_usec;
1813     uint32_t export_time_sec;
1814
1815     get_export_time_now(&export_time_usec, &export_time_sec);
1816     dpif_ipfix_cache_expire(exporter, forced_end, export_time_usec,
1817                             export_time_sec);
1818 }
1819
1820 void
1821 dpif_ipfix_run(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
1822 {
1823     uint64_t export_time_usec;
1824     uint32_t export_time_sec;
1825     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
1826
1827     ovs_mutex_lock(&mutex);
1828     get_export_time_now(&export_time_usec, &export_time_sec);
1829     if (di->bridge_exporter.probability > 0) {  /* Bridge exporter enabled. */
1830       dpif_ipfix_cache_expire(
1831           &di->bridge_exporter.exporter, false, export_time_usec,
1832           export_time_sec);
1833     }
1834     HMAP_FOR_EACH (flow_exporter_node, node, &di->flow_exporter_map) {
1835         dpif_ipfix_cache_expire(
1836             &flow_exporter_node->exporter.exporter, false, export_time_usec,
1837             export_time_sec);
1838     }
1839     ovs_mutex_unlock(&mutex);
1840 }
1841
1842 void
1843 dpif_ipfix_wait(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
1844 {
1845     long long int next_timeout_msec = LLONG_MAX;
1846     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
1847
1848     ovs_mutex_lock(&mutex);
1849     if (di->bridge_exporter.probability > 0) {  /* Bridge exporter enabled. */
1850         if (ipfix_cache_next_timeout_msec(
1851                 &di->bridge_exporter.exporter, &next_timeout_msec)) {
1852             poll_timer_wait_until(next_timeout_msec);
1853         }
1854     }
1855     HMAP_FOR_EACH (flow_exporter_node, node, &di->flow_exporter_map) {
1856         if (ipfix_cache_next_timeout_msec(
1857                 &flow_exporter_node->exporter.exporter, &next_timeout_msec)) {
1858             poll_timer_wait_until(next_timeout_msec);
1859         }
1860     }
1861     ovs_mutex_unlock(&mutex);
1862 }