ofproto/ofproto-dpif-ipfix.c

   1 /*
   2  * Copyright (c) 2012, 2013, 2014 Nicira, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <config.h>
  18 #include "ofproto-dpif-ipfix.h"
  19 #include <sys/time.h>
  20 #include "byte-order.h"
  21 #include "collectors.h"
  22 #include "flow.h"
  23 #include "hash.h"
  24 #include "hmap.h"
  25 #include "list.h"
  26 #include "ofpbuf.h"
  27 #include "ofproto.h"
  28 #include "ofproto-dpif.h"
  29 #include "dp-packet.h"
  30 #include "packets.h"
  31 #include "poll-loop.h"
  32 #include "sset.h"
  33 #include "util.h"
  34 #include "timeval.h"
  35 #include "util.h"
  36 #include "openvswitch/vlog.h"
  37
  38 VLOG_DEFINE_THIS_MODULE(ipfix);
  39
  40 static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
  41 static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
  42
  43 /* Cf. IETF RFC 5101 Section 10.3.4. */
  44 #define IPFIX_DEFAULT_COLLECTOR_PORT 4739
  45
  46 /* The standard layer2SegmentId (ID 351) element is included in vDS to send
  47  * the VxLAN tunnel's VNI. It is 64-bit long, the most significant byte is
  48  * used to indicate the type of tunnel (0x01 = VxLAN, 0x02 = GRE) and the three
  49  * least significant bytes hold the value of the layer 2 overlay network
  50  * segment identifier: a 24-bit VxLAN tunnel's VNI or a 24-bit GRE tunnel's
  51  * TNI. This is not compatible with GRE-64 or STT, as implemented in OVS, as
  52  * their tunnel IDs are 64-bit.
  53  *
  54  * Two new enterprise information elements are defined which are similar to
  55  * laryerSegmentId but support 64-bit IDs:
  56  *     tunnelType (ID 891) and tunnelKey (ID 892).
  57  *
  58  * The enum dpif_ipfix_tunnel_type is to declare the types supported in the
  59  * tunnelType element.
  60  * The number of ipfix tunnel types includes two reserverd types: 0x04 and 0x06.
  61  */
  62 enum dpif_ipfix_tunnel_type {
  63     DPIF_IPFIX_TUNNEL_UNKNOWN = 0x00,
  64     DPIF_IPFIX_TUNNEL_VXLAN = 0x01,
  65     DPIF_IPFIX_TUNNEL_GRE = 0x02,
  66     DPIF_IPFIX_TUNNEL_LISP = 0x03,
  67     DPIF_IPFIX_TUNNEL_STT = 0x04,
  68     DPIF_IPFIX_TUNNEL_IPSEC_GRE = 0x05,
  69     DPIF_IPFIX_TUNNEL_GENEVE = 0x07,
  70     NUM_DPIF_IPFIX_TUNNEL
  71 };
  72
  73 struct dpif_ipfix_port {
  74     struct hmap_node hmap_node; /* In struct dpif_ipfix's "tunnel_ports" hmap. */
  75     struct ofport *ofport;      /* To retrieve port stats. */
  76     odp_port_t odp_port;
  77     enum dpif_ipfix_tunnel_type tunnel_type;
  78     uint8_t tunnel_key_length;
  79 };
  80
  81 struct dpif_ipfix_exporter {
  82     struct collectors *collectors;
  83     uint32_t seq_number;
  84     time_t last_template_set_time;
  85     struct hmap cache_flow_key_map;  /* ipfix_flow_cache_entry. */
  86     struct ovs_list cache_flow_start_timestamp_list;  /* ipfix_flow_cache_entry. */
  87     uint32_t cache_active_timeout;  /* In seconds. */
  88     uint32_t cache_max_flows;
  89 };
  90
  91 struct dpif_ipfix_bridge_exporter {
  92     struct dpif_ipfix_exporter exporter;
  93     struct ofproto_ipfix_bridge_exporter_options *options;
  94     uint32_t probability;
  95 };
  96
  97 struct dpif_ipfix_flow_exporter {
  98     struct dpif_ipfix_exporter exporter;
  99     struct ofproto_ipfix_flow_exporter_options *options;
 100 };
 101
 102 struct dpif_ipfix_flow_exporter_map_node {
 103     struct hmap_node node;
 104     struct dpif_ipfix_flow_exporter exporter;
 105 };
 106
 107 struct dpif_ipfix {
 108     struct dpif_ipfix_bridge_exporter bridge_exporter;
 109     struct hmap flow_exporter_map;  /* dpif_ipfix_flow_exporter_map_node. */
 110     struct hmap tunnel_ports;       /* Contains "struct dpif_ipfix_port"s.
 111                                      * It makes tunnel port lookups faster in
 112                                      * sampling upcalls. */
 113     struct ovs_refcount ref_cnt;
 114 };
 115
 116 #define IPFIX_VERSION 0x000a
 117
 118 /* When using UDP, IPFIX Template Records must be re-sent regularly.
 119  * The standard default interval is 10 minutes (600 seconds).
 120  * Cf. IETF RFC 5101 Section 10.3.6. */
 121 #define IPFIX_TEMPLATE_INTERVAL 600
 122
 123 /* Cf. IETF RFC 5101 Section 3.1. */
 124 OVS_PACKED(
 125 struct ipfix_header {
 126     ovs_be16 version;  /* IPFIX_VERSION. */
 127     ovs_be16 length;  /* Length in bytes including this header. */
 128     ovs_be32 export_time;  /* Seconds since the epoch. */
 129     ovs_be32 seq_number;  /* Message sequence number. */
 130     ovs_be32 obs_domain_id;  /* Observation Domain ID. */
 131 });
 132 BUILD_ASSERT_DECL(sizeof(struct ipfix_header) == 16);
 133
 134 #define IPFIX_SET_ID_TEMPLATE 2
 135 #define IPFIX_SET_ID_OPTION_TEMPLATE 3
 136
 137 /* Cf. IETF RFC 5101 Section 3.3.2. */
 138 OVS_PACKED(
 139 struct ipfix_set_header {
 140     ovs_be16 set_id;  /* IPFIX_SET_ID_* or valid template ID for Data Sets. */
 141     ovs_be16 length;  /* Length of the set in bytes including header. */
 142 });
 143 BUILD_ASSERT_DECL(sizeof(struct ipfix_set_header) == 4);
 144
 145 /* Alternatives for templates at each layer.  A template is defined by
 146  * a combination of one value for each layer. */
 147 enum ipfix_proto_l2 {
 148     IPFIX_PROTO_L2_ETH = 0,  /* No VLAN. */
 149     IPFIX_PROTO_L2_VLAN,
 150     NUM_IPFIX_PROTO_L2
 151 };
 152 enum ipfix_proto_l3 {
 153     IPFIX_PROTO_L3_UNKNOWN = 0,
 154     IPFIX_PROTO_L3_IPV4,
 155     IPFIX_PROTO_L3_IPV6,
 156     NUM_IPFIX_PROTO_L3
 157 };
 158 enum ipfix_proto_l4 {
 159     IPFIX_PROTO_L4_UNKNOWN = 0,
 160     IPFIX_PROTO_L4_TCP_UDP_SCTP,
 161     IPFIX_PROTO_L4_ICMP,
 162     NUM_IPFIX_PROTO_L4
 163 };
 164 enum ipfix_proto_tunnel {
 165     IPFIX_PROTO_NOT_TUNNELED = 0,
 166     IPFIX_PROTO_TUNNELED,  /* Support gre, lisp and vxlan. */
 167     NUM_IPFIX_PROTO_TUNNEL
 168 };
 169
 170 /* Any Template ID > 255 is usable for Template Records. */
 171 #define IPFIX_TEMPLATE_ID_MIN 256
 172
 173 /* Cf. IETF RFC 5101 Section 3.4.1. */
 174 OVS_PACKED(
 175 struct ipfix_template_record_header {
 176     ovs_be16 template_id;
 177     ovs_be16 field_count;
 178 });
 179 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_record_header) == 4);
 180
 181 enum ipfix_entity_id {
 182 /* standard IPFIX elements */
 183 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ID_##ENUM = ID,
 184 #include "ofproto/ipfix-entities.def"
 185 /* non-standard IPFIX elements */
 186 #define IPFIX_SET_ENTERPRISE(v) (((v) | 0x8000))
 187 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 188     IPFIX_ENTITY_ID_##ENUM = IPFIX_SET_ENTERPRISE(ID),
 189 #include "ofproto/ipfix-enterprise-entities.def"
 190 };
 191
 192 enum ipfix_entity_size {
 193 /* standard IPFIX elements */
 194 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 195 #include "ofproto/ipfix-entities.def"
 196 /* non-standard IPFIX elements */
 197 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 198     IPFIX_ENTITY_SIZE_##ENUM = SIZE,
 199 #include "ofproto/ipfix-enterprise-entities.def"
 200 };
 201
 202 enum ipfix_entity_enterprise {
 203 /* standard IPFIX elements */
 204 #define IPFIX_ENTITY(ENUM, ID, SIZE, NAME)  IPFIX_ENTITY_ENTERPRISE_##ENUM = 0,
 205 #include "ofproto/ipfix-entities.def"
 206 /* non-standard IPFIX elements */
 207 #define IPFIX_ENTERPRISE_ENTITY(ENUM, ID, SIZE, NAME, ENTERPRISE) \
 208     IPFIX_ENTITY_ENTERPRISE_##ENUM = ENTERPRISE,
 209 #include "ofproto/ipfix-enterprise-entities.def"
 210 };
 211
 212 OVS_PACKED(
 213 struct ipfix_template_field_specifier {
 214     ovs_be16 element_id;  /* IPFIX_ENTITY_ID_*. */
 215     ovs_be16 field_length;  /* Length of the field's value, in bytes.
 216                              * For Variable-Length element, it should be 65535.
 217                              */
 218     ovs_be32 enterprise;  /* Enterprise number */
 219 });
 220 BUILD_ASSERT_DECL(sizeof(struct ipfix_template_field_specifier) == 8);
 221
 222 /* Cf. IETF RFC 5102 Section 5.11.6. */
 223 enum ipfix_flow_direction {
 224     INGRESS_FLOW = 0x00,
 225     EGRESS_FLOW = 0x01
 226 };
 227
 228 /* Part of data record flow key for common metadata and Ethernet entities. */
 229 OVS_PACKED(
 230 struct ipfix_data_record_flow_key_common {
 231     ovs_be32 observation_point_id;  /* OBSERVATION_POINT_ID */
 232     uint8_t flow_direction;  /* FLOW_DIRECTION */
 233     uint8_t source_mac_address[ETH_ADDR_LEN]; /* SOURCE_MAC_ADDRESS */
 234     uint8_t destination_mac_address[ETH_ADDR_LEN]; /* DESTINATION_MAC_ADDRESS */
 235     ovs_be16 ethernet_type;  /* ETHERNET_TYPE */
 236     uint8_t ethernet_header_length;  /* ETHERNET_HEADER_LENGTH */
 237 });
 238 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_common) == 20);
 239
 240 /* Part of data record flow key for VLAN entities. */
 241 OVS_PACKED(
 242 struct ipfix_data_record_flow_key_vlan {
 243     ovs_be16 vlan_id;  /* VLAN_ID */
 244     ovs_be16 dot1q_vlan_id;  /* DOT1Q_VLAN_ID */
 245     uint8_t dot1q_priority;  /* DOT1Q_PRIORITY */
 246 });
 247 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_vlan) == 5);
 248
 249 /* Part of data record flow key for IP entities. */
 250 /* XXX: Replace IP_TTL with MINIMUM_TTL and MAXIMUM_TTL? */
 251 OVS_PACKED(
 252 struct ipfix_data_record_flow_key_ip {
 253     uint8_t ip_version;  /* IP_VERSION */
 254     uint8_t ip_ttl;  /* IP_TTL */
 255     uint8_t protocol_identifier;  /* PROTOCOL_IDENTIFIER */
 256     uint8_t ip_diff_serv_code_point;  /* IP_DIFF_SERV_CODE_POINT */
 257     uint8_t ip_precedence;  /* IP_PRECEDENCE */
 258     uint8_t ip_class_of_service;  /* IP_CLASS_OF_SERVICE */
 259 });
 260 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ip) == 6);
 261
 262 /* Part of data record flow key for IPv4 entities. */
 263 OVS_PACKED(
 264 struct ipfix_data_record_flow_key_ipv4 {
 265     ovs_be32 source_ipv4_address;  /* SOURCE_IPV4_ADDRESS */
 266     ovs_be32 destination_ipv4_address;  /* DESTINATION_IPV4_ADDRESS */
 267 });
 268 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ipv4) == 8);
 269
 270 /* Part of data record flow key for IPv6 entities. */
 271 OVS_PACKED(
 272 struct ipfix_data_record_flow_key_ipv6 {
 273     uint8_t source_ipv6_address[16];  /* SOURCE_IPV6_ADDRESS */
 274     uint8_t destination_ipv6_address[16];  /* DESTINATION_IPV6_ADDRESS */
 275     ovs_be32 flow_label_ipv6;  /* FLOW_LABEL_IPV6 */
 276 });
 277 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_ipv6) == 36);
 278
 279 /* Part of data record flow key for TCP/UDP/SCTP entities. */
 280 OVS_PACKED(
 281 struct ipfix_data_record_flow_key_transport {
 282     ovs_be16 source_transport_port;  /* SOURCE_TRANSPORT_PORT */
 283     ovs_be16 destination_transport_port;  /* DESTINATION_TRANSPORT_PORT */
 284 });
 285 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_transport) == 4);
 286
 287 /* Part of data record flow key for ICMP entities. */
 288 OVS_PACKED(
 289 struct ipfix_data_record_flow_key_icmp {
 290     uint8_t icmp_type;  /* ICMP_TYPE_IPV4 / ICMP_TYPE_IPV6 */
 291     uint8_t icmp_code;  /* ICMP_CODE_IPV4 / ICMP_CODE_IPV6 */
 292 });
 293 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_icmp) == 2);
 294
 295 /* For the tunnel type that is on the top of IPSec, the protocol identifier
 296  * of the upper tunnel type is used.
 297  */
 298 static uint8_t tunnel_protocol[NUM_DPIF_IPFIX_TUNNEL] = {
 299     0,              /* reserved */
 300     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_VXLAN */
 301     IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_GRE */
 302     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_LISP*/
 303     IPPROTO_TCP,    /* DPIF_IPFIX_TUNNEL_STT*/
 304     IPPROTO_GRE,    /* DPIF_IPFIX_TUNNEL_IPSEC_GRE */
 305     0          ,    /* reserved */
 306     IPPROTO_UDP,    /* DPIF_IPFIX_TUNNEL_GENEVE*/
 307 };
 308
 309 OVS_PACKED(
 310 struct ipfix_data_record_flow_key_tunnel {
 311     ovs_be32 tunnel_source_ipv4_address;  /* TUNNEL_SOURCE_IPV4_ADDRESS */
 312     ovs_be32 tunnel_destination_ipv4_address;  /* TUNNEL_DESTINATION_IPV4_ADDRESS */
 313     uint8_t tunnel_protocol_identifier;  /* TUNNEL_PROTOCOL_IDENTIFIER */
 314     ovs_be16 tunnel_source_transport_port;  /* TUNNEL_SOURCE_TRANSPORT_PORT */
 315     ovs_be16 tunnel_destination_transport_port;  /* TUNNEL_DESTINATION_TRANSPORT_PORT */
 316     uint8_t tunnel_type;  /* TUNNEL_TYPE */
 317     uint8_t tunnel_key_length;  /* length of TUNNEL_KEY */
 318     uint8_t tunnel_key[];  /* data of  TUNNEL_KEY */
 319 });
 320 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_flow_key_tunnel) == 15);
 321
 322 /* Cf. IETF RFC 5102 Section 5.11.3. */
 323 enum ipfix_flow_end_reason {
 324     IDLE_TIMEOUT = 0x01,
 325     ACTIVE_TIMEOUT = 0x02,
 326     END_OF_FLOW_DETECTED = 0x03,
 327     FORCED_END = 0x04,
 328     LACK_OF_RESOURCES = 0x05
 329 };
 330
 331 /* Part of data record for common aggregated elements. */
 332 OVS_PACKED(
 333 struct ipfix_data_record_aggregated_common {
 334     ovs_be32 flow_start_delta_microseconds; /* FLOW_START_DELTA_MICROSECONDS */
 335     ovs_be32 flow_end_delta_microseconds; /* FLOW_END_DELTA_MICROSECONDS */
 336     ovs_be64 packet_delta_count;  /* PACKET_DELTA_COUNT */
 337     ovs_be64 layer2_octet_delta_count;  /* LAYER2_OCTET_DELTA_COUNT */
 338     uint8_t flow_end_reason;  /* FLOW_END_REASON */
 339 });
 340 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_common) == 25);
 341
 342 /* Part of data record for IP aggregated elements. */
 343 OVS_PACKED(
 344 struct ipfix_data_record_aggregated_ip {
 345     ovs_be64 octet_delta_count;  /* OCTET_DELTA_COUNT */
 346     ovs_be64 octet_delta_sum_of_squares;  /* OCTET_DELTA_SUM_OF_SQUARES */
 347     ovs_be64 minimum_ip_total_length;  /* MINIMUM_IP_TOTAL_LENGTH */
 348     ovs_be64 maximum_ip_total_length;  /* MAXIMUM_IP_TOTAL_LENGTH */
 349 });
 350 BUILD_ASSERT_DECL(sizeof(struct ipfix_data_record_aggregated_ip) == 32);
 351
 352 /*
 353  * support tunnel key for:
 354  * VxLAN: 24-bit VIN,
 355  * GRE: 32- or 64-bit key,
 356  * LISP: 24-bit instance ID
 357  * STT: 64-bit key
 358  */
 359 #define MAX_TUNNEL_KEY_LEN 8
 360
 361 #define MAX_FLOW_KEY_LEN                                        \
 362     (sizeof(struct ipfix_data_record_flow_key_common)           \
 363      + sizeof(struct ipfix_data_record_flow_key_vlan)           \
 364      + sizeof(struct ipfix_data_record_flow_key_ip)             \
 365      + MAX(sizeof(struct ipfix_data_record_flow_key_ipv4),      \
 366            sizeof(struct ipfix_data_record_flow_key_ipv6))      \
 367      + MAX(sizeof(struct ipfix_data_record_flow_key_icmp),      \
 368            sizeof(struct ipfix_data_record_flow_key_transport)) \
 369      + sizeof(struct ipfix_data_record_flow_key_tunnel)         \
 370      + MAX_TUNNEL_KEY_LEN)
 371
 372 #define MAX_DATA_RECORD_LEN                                 \
 373     (MAX_FLOW_KEY_LEN                                       \
 374      + sizeof(struct ipfix_data_record_aggregated_common)   \
 375      + sizeof(struct ipfix_data_record_aggregated_ip))
 376
 377 /* Max length of a data set.  To simplify the implementation, each
 378  * data record is sent in a separate data set, so each data set
 379  * contains at most one data record. */
 380 #define MAX_DATA_SET_LEN             \
 381     (sizeof(struct ipfix_set_header) \
 382      + MAX_DATA_RECORD_LEN)
 383
 384 /* Max length of an IPFIX message. Arbitrarily set to accommodate low
 385  * MTU. */
 386 #define MAX_MESSAGE_LEN 1024
 387
 388 /* Cache structures. */
 389
 390 /* Flow key. */
 391 struct ipfix_flow_key {
 392     uint32_t obs_domain_id;
 393     uint16_t template_id;
 394     size_t flow_key_msg_part_size;
 395     uint64_t flow_key_msg_part[DIV_ROUND_UP(MAX_FLOW_KEY_LEN, 8)];
 396 };
 397
 398 /* Flow cache entry. */
 399 struct ipfix_flow_cache_entry {
 400     struct hmap_node flow_key_map_node;
 401     struct ovs_list cache_flow_start_timestamp_list_node;
 402     struct ipfix_flow_key flow_key;
 403     /* Common aggregated elements. */
 404     uint64_t flow_start_timestamp_usec;
 405     uint64_t flow_end_timestamp_usec;
 406     uint64_t packet_delta_count;
 407     uint64_t layer2_octet_delta_count;
 408     uint64_t octet_delta_count;
 409     uint64_t octet_delta_sum_of_squares;  /* 0 if not IP. */
 410     uint16_t minimum_ip_total_length;  /* 0 if not IP. */
 411     uint16_t maximum_ip_total_length;  /* 0 if not IP. */
 412 };
 413
 414 static void dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *, bool,
 415                                     const uint64_t, const uint32_t);
 416
 417 static void get_export_time_now(uint64_t *, uint32_t *);
 418
 419 static void dpif_ipfix_cache_expire_now(struct dpif_ipfix_exporter *, bool);
 420
 421 static bool
 422 ofproto_ipfix_bridge_exporter_options_equal(
 423     const struct ofproto_ipfix_bridge_exporter_options *a,
 424     const struct ofproto_ipfix_bridge_exporter_options *b)
 425 {
 426     return (a->obs_domain_id == b->obs_domain_id
 427             && a->obs_point_id == b->obs_point_id
 428             && a->sampling_rate == b->sampling_rate
 429             && a->cache_active_timeout == b->cache_active_timeout
 430             && a->cache_max_flows == b->cache_max_flows
 431             && a->enable_tunnel_sampling == b->enable_tunnel_sampling
 432             && a->enable_input_sampling == b->enable_input_sampling
 433             && a->enable_output_sampling == b->enable_output_sampling
 434             && sset_equals(&a->targets, &b->targets));
 435 }
 436
 437 static struct ofproto_ipfix_bridge_exporter_options *
 438 ofproto_ipfix_bridge_exporter_options_clone(
 439     const struct ofproto_ipfix_bridge_exporter_options *old)
 440 {
 441     struct ofproto_ipfix_bridge_exporter_options *new =
 442         xmemdup(old, sizeof *old);
 443     sset_clone(&new->targets, &old->targets);
 444     return new;
 445 }
 446
 447 static void
 448 ofproto_ipfix_bridge_exporter_options_destroy(
 449     struct ofproto_ipfix_bridge_exporter_options *options)
 450 {
 451     if (options) {
 452         sset_destroy(&options->targets);
 453         free(options);
 454     }
 455 }
 456
 457 static bool
 458 ofproto_ipfix_flow_exporter_options_equal(
 459     const struct ofproto_ipfix_flow_exporter_options *a,
 460     const struct ofproto_ipfix_flow_exporter_options *b)
 461 {
 462     return (a->collector_set_id == b->collector_set_id
 463             && a->cache_active_timeout == b->cache_active_timeout
 464             && a->cache_max_flows == b->cache_max_flows
 465             && sset_equals(&a->targets, &b->targets));
 466 }
 467
 468 static struct ofproto_ipfix_flow_exporter_options *
 469 ofproto_ipfix_flow_exporter_options_clone(
 470     const struct ofproto_ipfix_flow_exporter_options *old)
 471 {
 472     struct ofproto_ipfix_flow_exporter_options *new =
 473         xmemdup(old, sizeof *old);
 474     sset_clone(&new->targets, &old->targets);
 475     return new;
 476 }
 477
 478 static void
 479 ofproto_ipfix_flow_exporter_options_destroy(
 480     struct ofproto_ipfix_flow_exporter_options *options)
 481 {
 482     if (options) {
 483         sset_destroy(&options->targets);
 484         free(options);
 485     }
 486 }
 487
 488 static void
 489 dpif_ipfix_exporter_init(struct dpif_ipfix_exporter *exporter)
 490 {
 491     exporter->collectors = NULL;
 492     exporter->seq_number = 1;
 493     exporter->last_template_set_time = TIME_MIN;
 494     hmap_init(&exporter->cache_flow_key_map);
 495     list_init(&exporter->cache_flow_start_timestamp_list);
 496     exporter->cache_active_timeout = 0;
 497     exporter->cache_max_flows = 0;
 498 }
 499
 500 static void
 501 dpif_ipfix_exporter_clear(struct dpif_ipfix_exporter *exporter)
 502 {
 503     /* Flush the cache with flow end reason "forced end." */
 504     dpif_ipfix_cache_expire_now(exporter, true);
 505
 506     collectors_destroy(exporter->collectors);
 507     exporter->collectors = NULL;
 508     exporter->seq_number = 1;
 509     exporter->last_template_set_time = TIME_MIN;
 510     exporter->cache_active_timeout = 0;
 511     exporter->cache_max_flows = 0;
 512 }
 513
 514 static void
 515 dpif_ipfix_exporter_destroy(struct dpif_ipfix_exporter *exporter)
 516 {
 517     dpif_ipfix_exporter_clear(exporter);
 518     hmap_destroy(&exporter->cache_flow_key_map);
 519 }
 520
 521 static bool
 522 dpif_ipfix_exporter_set_options(struct dpif_ipfix_exporter *exporter,
 523                                 const struct sset *targets,
 524                                 const uint32_t cache_active_timeout,
 525                                 const uint32_t cache_max_flows)
 526 {
 527     collectors_destroy(exporter->collectors);
 528     collectors_create(targets, IPFIX_DEFAULT_COLLECTOR_PORT,
 529                       &exporter->collectors);
 530     if (exporter->collectors == NULL) {
 531         VLOG_WARN_RL(&rl, "no collectors could be initialized, "
 532                      "IPFIX exporter disabled");
 533         dpif_ipfix_exporter_clear(exporter);
 534         return false;
 535     }
 536     exporter->cache_active_timeout = cache_active_timeout;
 537     exporter->cache_max_flows = cache_max_flows;
 538     return true;
 539 }
 540
 541 static struct dpif_ipfix_port *
 542 dpif_ipfix_find_port(const struct dpif_ipfix *di,
 543                      odp_port_t odp_port) OVS_REQUIRES(mutex)
 544 {
 545     struct dpif_ipfix_port *dip;
 546
 547     HMAP_FOR_EACH_IN_BUCKET (dip, hmap_node, hash_odp_port(odp_port),
 548                              &di->tunnel_ports) {
 549         if (dip->odp_port == odp_port) {
 550             return dip;
 551         }
 552     }
 553     return NULL;
 554 }
 555
 556 static void
 557 dpif_ipfix_del_port(struct dpif_ipfix *di,
 558                       struct dpif_ipfix_port *dip)
 559     OVS_REQUIRES(mutex)
 560 {
 561     hmap_remove(&di->tunnel_ports, &dip->hmap_node);
 562     free(dip);
 563 }
 564
 565 void
 566 dpif_ipfix_add_tunnel_port(struct dpif_ipfix *di, struct ofport *ofport,
 567                            odp_port_t odp_port) OVS_EXCLUDED(mutex)
 568 {
 569     struct dpif_ipfix_port *dip;
 570     const char *type;
 571
 572     ovs_mutex_lock(&mutex);
 573     dip = dpif_ipfix_find_port(di, odp_port);
 574     if (dip) {
 575         dpif_ipfix_del_port(di, dip);
 576     }
 577
 578     type = netdev_get_type(ofport->netdev);
 579     if (type == NULL) {
 580         goto out;
 581     }
 582
 583     /* Add to table of tunnel ports. */
 584     dip = xmalloc(sizeof *dip);
 585     dip->ofport = ofport;
 586     dip->odp_port = odp_port;
 587     if (strcmp(type, "gre") == 0) {
 588         /* 32-bit key gre */
 589         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
 590         dip->tunnel_key_length = 4;
 591     } else if (strcmp(type, "gre64") == 0) {
 592         /* 64-bit key gre */
 593         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GRE;
 594         dip->tunnel_key_length = 8;
 595     } else if (strcmp(type, "ipsec_gre") == 0) {
 596         /* 32-bit key ipsec_gre */
 597         dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
 598         dip->tunnel_key_length = 4;
 599     } else if (strcmp(type, "ipsec_gre64") == 0) {
 600         /* 64-bit key ipsec_gre */
 601         dip->tunnel_type = DPIF_IPFIX_TUNNEL_IPSEC_GRE;
 602         dip->tunnel_key_length = 8;
 603     } else if (strcmp(type, "vxlan") == 0) {
 604         dip->tunnel_type = DPIF_IPFIX_TUNNEL_VXLAN;
 605         dip->tunnel_key_length = 3;
 606     } else if (strcmp(type, "lisp") == 0) {
 607         dip->tunnel_type = DPIF_IPFIX_TUNNEL_LISP;
 608         dip->tunnel_key_length = 3;
 609     } else if (strcmp(type, "geneve") == 0) {
 610         dip->tunnel_type = DPIF_IPFIX_TUNNEL_GENEVE;
 611         dip->tunnel_key_length = 3;
 612     } else if (strcmp(type, "stt") == 0) {
 613         dip->tunnel_type = DPIF_IPFIX_TUNNEL_STT;
 614         dip->tunnel_key_length = 8;
 615     } else {
 616         free(dip);
 617         goto out;
 618     }
 619     hmap_insert(&di->tunnel_ports, &dip->hmap_node, hash_odp_port(odp_port));
 620
 621 out:
 622     ovs_mutex_unlock(&mutex);
 623 }
 624
 625 void
 626 dpif_ipfix_del_tunnel_port(struct dpif_ipfix *di, odp_port_t odp_port)
 627     OVS_EXCLUDED(mutex)
 628 {
 629     struct dpif_ipfix_port *dip;
 630     ovs_mutex_lock(&mutex);
 631     dip = dpif_ipfix_find_port(di, odp_port);
 632     if (dip) {
 633         dpif_ipfix_del_port(di, dip);
 634     }
 635     ovs_mutex_unlock(&mutex);
 636 }
 637
 638 bool
 639 dpif_ipfix_get_tunnel_port(const struct dpif_ipfix *di, odp_port_t odp_port)
 640     OVS_EXCLUDED(mutex)
 641 {
 642     struct dpif_ipfix_port *dip;
 643     ovs_mutex_lock(&mutex);
 644     dip = dpif_ipfix_find_port(di, odp_port);
 645     ovs_mutex_unlock(&mutex);
 646     return dip != NULL;
 647 }
 648
 649 static void
 650 dpif_ipfix_bridge_exporter_init(struct dpif_ipfix_bridge_exporter *exporter)
 651 {
 652     dpif_ipfix_exporter_init(&exporter->exporter);
 653     exporter->options = NULL;
 654     exporter->probability = 0;
 655 }
 656
 657 static void
 658 dpif_ipfix_bridge_exporter_clear(struct dpif_ipfix_bridge_exporter *exporter)
 659 {
 660     dpif_ipfix_exporter_clear(&exporter->exporter);
 661     ofproto_ipfix_bridge_exporter_options_destroy(exporter->options);
 662     exporter->options = NULL;
 663     exporter->probability = 0;
 664 }
 665
 666 static void
 667 dpif_ipfix_bridge_exporter_destroy(struct dpif_ipfix_bridge_exporter *exporter)
 668 {
 669     dpif_ipfix_bridge_exporter_clear(exporter);
 670     dpif_ipfix_exporter_destroy(&exporter->exporter);
 671 }
 672
 673 static void
 674 dpif_ipfix_bridge_exporter_set_options(
 675     struct dpif_ipfix_bridge_exporter *exporter,
 676     const struct ofproto_ipfix_bridge_exporter_options *options)
 677 {
 678     bool options_changed;
 679
 680     if (!options || sset_is_empty(&options->targets)) {
 681         /* No point in doing any work if there are no targets. */
 682         dpif_ipfix_bridge_exporter_clear(exporter);
 683         return;
 684     }
 685
 686     options_changed = (
 687         !exporter->options
 688         || !ofproto_ipfix_bridge_exporter_options_equal(
 689             options, exporter->options));
 690
 691     /* Configure collectors if options have changed or if we're
 692      * shortchanged in collectors (which indicates that opening one or
 693      * more of the configured collectors failed, so that we should
 694      * retry). */
 695     if (options_changed
 696         || collectors_count(exporter->exporter.collectors)
 697             < sset_count(&options->targets)) {
 698         if (!dpif_ipfix_exporter_set_options(
 699                 &exporter->exporter, &options->targets,
 700                 options->cache_active_timeout, options->cache_max_flows)) {
 701             return;
 702         }
 703     }
 704
 705     /* Avoid reconfiguring if options didn't change. */
 706     if (!options_changed) {
 707         return;
 708     }
 709
 710     ofproto_ipfix_bridge_exporter_options_destroy(exporter->options);
 711     exporter->options = ofproto_ipfix_bridge_exporter_options_clone(options);
 712     exporter->probability =
 713         MAX(1, UINT32_MAX / exporter->options->sampling_rate);
 714
 715     /* Run over the cache as some entries might have expired after
 716      * changing the timeouts. */
 717     dpif_ipfix_cache_expire_now(&exporter->exporter, false);
 718 }
 719
 720 static struct dpif_ipfix_flow_exporter_map_node*
 721 dpif_ipfix_find_flow_exporter_map_node(
 722     const struct dpif_ipfix *di, const uint32_t collector_set_id)
 723     OVS_REQUIRES(mutex)
 724 {
 725     struct dpif_ipfix_flow_exporter_map_node *exporter_node;
 726
 727     HMAP_FOR_EACH_WITH_HASH (exporter_node, node,
 728                              hash_int(collector_set_id, 0),
 729                              &di->flow_exporter_map) {
 730         if (exporter_node->exporter.options->collector_set_id
 731             == collector_set_id) {
 732             return exporter_node;
 733         }
 734     }
 735
 736     return NULL;
 737 }
 738
 739 static void
 740 dpif_ipfix_flow_exporter_init(struct dpif_ipfix_flow_exporter *exporter)
 741 {
 742     dpif_ipfix_exporter_init(&exporter->exporter);
 743     exporter->options = NULL;
 744 }
 745
 746 static void
 747 dpif_ipfix_flow_exporter_clear(struct dpif_ipfix_flow_exporter *exporter)
 748 {
 749     dpif_ipfix_exporter_clear(&exporter->exporter);
 750     ofproto_ipfix_flow_exporter_options_destroy(exporter->options);
 751     exporter->options = NULL;
 752 }
 753
 754 static void
 755 dpif_ipfix_flow_exporter_destroy(struct dpif_ipfix_flow_exporter *exporter)
 756 {
 757     dpif_ipfix_flow_exporter_clear(exporter);
 758     dpif_ipfix_exporter_destroy(&exporter->exporter);
 759 }
 760
 761 static bool
 762 dpif_ipfix_flow_exporter_set_options(
 763     struct dpif_ipfix_flow_exporter *exporter,
 764     const struct ofproto_ipfix_flow_exporter_options *options)
 765 {
 766     bool options_changed;
 767
 768     if (sset_is_empty(&options->targets)) {
 769         /* No point in doing any work if there are no targets. */
 770         dpif_ipfix_flow_exporter_clear(exporter);
 771         return true;
 772     }
 773
 774     options_changed = (
 775         !exporter->options
 776         || !ofproto_ipfix_flow_exporter_options_equal(
 777             options, exporter->options));
 778
 779     /* Configure collectors if options have changed or if we're
 780      * shortchanged in collectors (which indicates that opening one or
 781      * more of the configured collectors failed, so that we should
 782      * retry). */
 783     if (options_changed
 784         || collectors_count(exporter->exporter.collectors)
 785             < sset_count(&options->targets)) {
 786         if (!dpif_ipfix_exporter_set_options(
 787                 &exporter->exporter, &options->targets,
 788                 options->cache_active_timeout, options->cache_max_flows)) {
 789             return false;
 790         }
 791     }
 792
 793     /* Avoid reconfiguring if options didn't change. */
 794     if (!options_changed) {
 795         return true;
 796     }
 797
 798     ofproto_ipfix_flow_exporter_options_destroy(exporter->options);
 799     exporter->options = ofproto_ipfix_flow_exporter_options_clone(options);
 800
 801     /* Run over the cache as some entries might have expired after
 802      * changing the timeouts. */
 803     dpif_ipfix_cache_expire_now(&exporter->exporter, false);
 804
 805     return true;
 806 }
 807
 808 void
 809 dpif_ipfix_set_options(
 810     struct dpif_ipfix *di,
 811     const struct ofproto_ipfix_bridge_exporter_options *bridge_exporter_options,
 812     const struct ofproto_ipfix_flow_exporter_options *flow_exporters_options,
 813     size_t n_flow_exporters_options) OVS_EXCLUDED(mutex)
 814 {
 815     int i;
 816     struct ofproto_ipfix_flow_exporter_options *options;
 817     struct dpif_ipfix_flow_exporter_map_node *node, *next;
 818     size_t n_broken_flow_exporters_options = 0;
 819
 820     ovs_mutex_lock(&mutex);
 821     dpif_ipfix_bridge_exporter_set_options(&di->bridge_exporter,
 822                                            bridge_exporter_options);
 823
 824     /* Add new flow exporters and update current flow exporters. */
 825     options = (struct ofproto_ipfix_flow_exporter_options *)
 826         flow_exporters_options;
 827     for (i = 0; i < n_flow_exporters_options; i++) {
 828         node = dpif_ipfix_find_flow_exporter_map_node(
 829             di, options->collector_set_id);
 830         if (!node) {
 831             node = xzalloc(sizeof *node);
 832             dpif_ipfix_flow_exporter_init(&node->exporter);
 833             hmap_insert(&di->flow_exporter_map, &node->node,
 834                         hash_int(options->collector_set_id, 0));
 835         }
 836         if (!dpif_ipfix_flow_exporter_set_options(&node->exporter, options)) {
 837             n_broken_flow_exporters_options++;
 838         }
 839         options++;
 840     }
 841
 842     ovs_assert(hmap_count(&di->flow_exporter_map) >=
 843                (n_flow_exporters_options - n_broken_flow_exporters_options));
 844
 845     /* Remove dropped flow exporters, if any needs to be removed. */
 846     if (hmap_count(&di->flow_exporter_map) > n_flow_exporters_options) {
 847         HMAP_FOR_EACH_SAFE (node, next, node, &di->flow_exporter_map) {
 848             /* This is slow but doesn't take any extra memory, and
 849              * this table is not supposed to contain many rows anyway. */
 850             options = (struct ofproto_ipfix_flow_exporter_options *)
 851                 flow_exporters_options;
 852             for (i = 0; i < n_flow_exporters_options; i++) {
 853               if (node->exporter.options->collector_set_id
 854                   == options->collector_set_id) {
 855                   break;
 856               }
 857               options++;
 858             }
 859             if (i == n_flow_exporters_options) {  // Not found.
 860                 hmap_remove(&di->flow_exporter_map, &node->node);
 861                 dpif_ipfix_flow_exporter_destroy(&node->exporter);
 862                 free(node);
 863             }
 864         }
 865     }
 866
 867     ovs_assert(hmap_count(&di->flow_exporter_map) ==
 868                (n_flow_exporters_options - n_broken_flow_exporters_options));
 869     ovs_mutex_unlock(&mutex);
 870 }
 871
 872 struct dpif_ipfix *
 873 dpif_ipfix_create(void)
 874 {
 875     struct dpif_ipfix *di;
 876     di = xzalloc(sizeof *di);
 877     dpif_ipfix_bridge_exporter_init(&di->bridge_exporter);
 878     hmap_init(&di->flow_exporter_map);
 879     hmap_init(&di->tunnel_ports);
 880     ovs_refcount_init(&di->ref_cnt);
 881     return di;
 882 }
 883
 884 struct dpif_ipfix *
 885 dpif_ipfix_ref(const struct dpif_ipfix *di_)
 886 {
 887     struct dpif_ipfix *di = CONST_CAST(struct dpif_ipfix *, di_);
 888     if (di) {
 889         ovs_refcount_ref(&di->ref_cnt);
 890     }
 891     return di;
 892 }
 893
 894 uint32_t
 895 dpif_ipfix_get_bridge_exporter_probability(const struct dpif_ipfix *di)
 896     OVS_EXCLUDED(mutex)
 897 {
 898     uint32_t ret;
 899     ovs_mutex_lock(&mutex);
 900     ret = di->bridge_exporter.probability;
 901     ovs_mutex_unlock(&mutex);
 902     return ret;
 903 }
 904
 905 bool
 906 dpif_ipfix_get_bridge_exporter_input_sampling(const struct dpif_ipfix *di)
 907     OVS_EXCLUDED(mutex)
 908 {
 909     bool ret = true;
 910     ovs_mutex_lock(&mutex);
 911     if (di->bridge_exporter.options) {
 912         ret = di->bridge_exporter.options->enable_input_sampling;
 913     }
 914     ovs_mutex_unlock(&mutex);
 915     return ret;
 916 }
 917
 918 bool
 919 dpif_ipfix_get_bridge_exporter_output_sampling(const struct dpif_ipfix *di)
 920     OVS_EXCLUDED(mutex)
 921 {
 922     bool ret = true;
 923     ovs_mutex_lock(&mutex);
 924     if (di->bridge_exporter.options) {
 925         ret = di->bridge_exporter.options->enable_output_sampling;
 926     }
 927     ovs_mutex_unlock(&mutex);
 928     return ret;
 929 }
 930
 931 bool
 932 dpif_ipfix_get_bridge_exporter_tunnel_sampling(const struct dpif_ipfix *di)
 933     OVS_EXCLUDED(mutex)
 934 {
 935     bool ret = false;
 936     ovs_mutex_lock(&mutex);
 937     if (di->bridge_exporter.options) {
 938         ret = di->bridge_exporter.options->enable_tunnel_sampling;
 939     }
 940     ovs_mutex_unlock(&mutex);
 941     return ret;
 942 }
 943
 944 static void
 945 dpif_ipfix_clear(struct dpif_ipfix *di) OVS_REQUIRES(mutex)
 946 {
 947     struct dpif_ipfix_flow_exporter_map_node *exp_node, *exp_next;
 948     struct dpif_ipfix_port *dip, *next;
 949
 950     dpif_ipfix_bridge_exporter_clear(&di->bridge_exporter);
 951
 952     HMAP_FOR_EACH_SAFE (exp_node, exp_next, node, &di->flow_exporter_map) {
 953         hmap_remove(&di->flow_exporter_map, &exp_node->node);
 954         dpif_ipfix_flow_exporter_destroy(&exp_node->exporter);
 955         free(exp_node);
 956     }
 957
 958     HMAP_FOR_EACH_SAFE (dip, next, hmap_node, &di->tunnel_ports) {
 959         dpif_ipfix_del_port(di, dip);
 960     }
 961 }
 962
 963 void
 964 dpif_ipfix_unref(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
 965 {
 966     if (di && ovs_refcount_unref_relaxed(&di->ref_cnt) == 1) {
 967         ovs_mutex_lock(&mutex);
 968         dpif_ipfix_clear(di);
 969         dpif_ipfix_bridge_exporter_destroy(&di->bridge_exporter);
 970         hmap_destroy(&di->flow_exporter_map);
 971         hmap_destroy(&di->tunnel_ports);
 972         free(di);
 973         ovs_mutex_unlock(&mutex);
 974     }
 975 }
 976
 977 static void
 978 ipfix_init_header(uint32_t export_time_sec, uint32_t seq_number,
 979                   uint32_t obs_domain_id, struct dp_packet *msg)
 980 {
 981     struct ipfix_header *hdr;
 982
 983     hdr = dp_packet_put_zeros(msg, sizeof *hdr);
 984     hdr->version = htons(IPFIX_VERSION);
 985     hdr->length = htons(sizeof *hdr);  /* Updated in ipfix_send_msg. */
 986     hdr->export_time = htonl(export_time_sec);
 987     hdr->seq_number = htonl(seq_number);
 988     hdr->obs_domain_id = htonl(obs_domain_id);
 989 }
 990
 991 static void
 992 ipfix_send_msg(const struct collectors *collectors, struct dp_packet *msg)
 993 {
 994     struct ipfix_header *hdr;
 995
 996     /* Adjust the length in the header. */
 997     hdr = dp_packet_data(msg);
 998     hdr->length = htons(dp_packet_size(msg));
 999
1000     collectors_send(collectors, dp_packet_data(msg), dp_packet_size(msg));
1001     dp_packet_set_size(msg, 0);
1002 }
1003
1004 static uint16_t
1005 ipfix_get_template_id(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
1006                       enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel)
1007 {
1008     uint16_t template_id;
1009     template_id = l2;
1010     template_id = template_id * NUM_IPFIX_PROTO_L3 + l3;
1011     template_id = template_id * NUM_IPFIX_PROTO_L4 + l4;
1012     template_id = template_id * NUM_IPFIX_PROTO_TUNNEL + tunnel;
1013     return IPFIX_TEMPLATE_ID_MIN + template_id;
1014 }
1015
1016 static void
1017 ipfix_define_template_entity(enum ipfix_entity_id id,
1018                              enum ipfix_entity_size size,
1019                              enum ipfix_entity_enterprise enterprise,
1020                              struct dp_packet *msg)
1021 {
1022     struct ipfix_template_field_specifier *field;
1023     size_t field_size;
1024
1025     if (enterprise) {
1026         field_size = sizeof *field;
1027     } else {
1028         /* No enterprise number */
1029         field_size = sizeof *field - sizeof(ovs_be32);
1030     }
1031     field = dp_packet_put_zeros(msg, field_size);
1032     field->element_id = htons(id);
1033     if (size) {
1034         field->field_length = htons(size);
1035     } else {
1036         /* RFC 5101, Section 7. Variable-Length Information Element */
1037         field->field_length = OVS_BE16_MAX;
1038     }
1039     if (enterprise) {
1040         field->enterprise = htonl(enterprise);
1041     }
1042
1043 }
1044
1045 static uint16_t
1046 ipfix_define_template_fields(enum ipfix_proto_l2 l2, enum ipfix_proto_l3 l3,
1047                              enum ipfix_proto_l4 l4, enum ipfix_proto_tunnel tunnel,
1048                              struct dp_packet *msg)
1049 {
1050     uint16_t count = 0;
1051
1052 #define DEF(ID) \
1053     { \
1054         ipfix_define_template_entity(IPFIX_ENTITY_ID_##ID, \
1055                                      IPFIX_ENTITY_SIZE_##ID, \
1056                                      IPFIX_ENTITY_ENTERPRISE_##ID, msg); \
1057         count++; \
1058     }
1059
1060     /* 1. Flow key. */
1061
1062     DEF(OBSERVATION_POINT_ID);
1063     DEF(FLOW_DIRECTION);
1064
1065     /* Common Ethernet entities. */
1066     DEF(SOURCE_MAC_ADDRESS);
1067     DEF(DESTINATION_MAC_ADDRESS);
1068     DEF(ETHERNET_TYPE);
1069     DEF(ETHERNET_HEADER_LENGTH);
1070
1071     if (l2 == IPFIX_PROTO_L2_VLAN) {
1072         DEF(VLAN_ID);
1073         DEF(DOT1Q_VLAN_ID);
1074         DEF(DOT1Q_PRIORITY);
1075     }
1076
1077     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1078         DEF(IP_VERSION);
1079         DEF(IP_TTL);
1080         DEF(PROTOCOL_IDENTIFIER);
1081         DEF(IP_DIFF_SERV_CODE_POINT);
1082         DEF(IP_PRECEDENCE);
1083         DEF(IP_CLASS_OF_SERVICE);
1084
1085         if (l3 == IPFIX_PROTO_L3_IPV4) {
1086             DEF(SOURCE_IPV4_ADDRESS);
1087             DEF(DESTINATION_IPV4_ADDRESS);
1088             if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1089                 DEF(SOURCE_TRANSPORT_PORT);
1090                 DEF(DESTINATION_TRANSPORT_PORT);
1091             } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1092                 DEF(ICMP_TYPE_IPV4);
1093                 DEF(ICMP_CODE_IPV4);
1094             }
1095         } else {  /* l3 == IPFIX_PROTO_L3_IPV6 */
1096             DEF(SOURCE_IPV6_ADDRESS);
1097             DEF(DESTINATION_IPV6_ADDRESS);
1098             DEF(FLOW_LABEL_IPV6);
1099             if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1100                 DEF(SOURCE_TRANSPORT_PORT);
1101                 DEF(DESTINATION_TRANSPORT_PORT);
1102             } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1103                 DEF(ICMP_TYPE_IPV6);
1104                 DEF(ICMP_CODE_IPV6);
1105             }
1106         }
1107     }
1108
1109     if (tunnel != IPFIX_PROTO_NOT_TUNNELED) {
1110         DEF(TUNNEL_SOURCE_IPV4_ADDRESS);
1111         DEF(TUNNEL_DESTINATION_IPV4_ADDRESS);
1112         DEF(TUNNEL_PROTOCOL_IDENTIFIER);
1113         DEF(TUNNEL_SOURCE_TRANSPORT_PORT);
1114         DEF(TUNNEL_DESTINATION_TRANSPORT_PORT);
1115         DEF(TUNNEL_TYPE);
1116         DEF(TUNNEL_KEY);
1117     }
1118
1119     /* 2. Flow aggregated data. */
1120
1121     DEF(FLOW_START_DELTA_MICROSECONDS);
1122     DEF(FLOW_END_DELTA_MICROSECONDS);
1123     DEF(PACKET_DELTA_COUNT);
1124     DEF(LAYER2_OCTET_DELTA_COUNT);
1125     DEF(FLOW_END_REASON);
1126
1127     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1128         DEF(OCTET_DELTA_COUNT);
1129         DEF(OCTET_DELTA_SUM_OF_SQUARES);
1130         DEF(MINIMUM_IP_TOTAL_LENGTH);
1131         DEF(MAXIMUM_IP_TOTAL_LENGTH);
1132     }
1133
1134
1135 #undef DEF
1136
1137     return count;
1138 }
1139
1140 static void
1141 ipfix_init_template_msg(void *msg_stub, uint32_t export_time_sec,
1142                         uint32_t seq_number, uint32_t obs_domain_id,
1143                         struct dp_packet *msg, size_t *set_hdr_offset)
1144 {
1145     struct ipfix_set_header *set_hdr;
1146
1147     dp_packet_use_stub(msg, msg_stub, sizeof msg_stub);
1148
1149     ipfix_init_header(export_time_sec, seq_number, obs_domain_id, msg);
1150     *set_hdr_offset = dp_packet_size(msg);
1151
1152     /* Add a Template Set. */
1153     set_hdr = dp_packet_put_zeros(msg, sizeof *set_hdr);
1154     set_hdr->set_id = htons(IPFIX_SET_ID_TEMPLATE);
1155 }
1156
1157 static void
1158 ipfix_send_template_msg(const struct collectors *collectors,
1159                         struct dp_packet *msg, size_t set_hdr_offset)
1160 {
1161     struct ipfix_set_header *set_hdr;
1162
1163     /* Send template message. */
1164     set_hdr = (struct ipfix_set_header*)
1165               ((uint8_t*)dp_packet_data(msg) + set_hdr_offset);
1166     set_hdr->length = htons(dp_packet_size(msg) - set_hdr_offset);
1167
1168     ipfix_send_msg(collectors, msg);
1169
1170     dp_packet_uninit(msg);
1171 }
1172
1173 static void
1174 ipfix_send_template_msgs(struct dpif_ipfix_exporter *exporter,
1175                          uint32_t export_time_sec, uint32_t obs_domain_id)
1176 {
1177     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
1178     struct dp_packet msg;
1179     size_t set_hdr_offset, tmpl_hdr_offset;
1180     struct ipfix_template_record_header *tmpl_hdr;
1181     uint16_t field_count;
1182     enum ipfix_proto_l2 l2;
1183     enum ipfix_proto_l3 l3;
1184     enum ipfix_proto_l4 l4;
1185     enum ipfix_proto_tunnel tunnel;
1186
1187     ipfix_init_template_msg(msg_stub, export_time_sec, exporter->seq_number,
1188                             obs_domain_id, &msg, &set_hdr_offset);
1189     /* Define one template for each possible combination of
1190      * protocols. */
1191     for (l2 = 0; l2 < NUM_IPFIX_PROTO_L2; l2++) {
1192         for (l3 = 0; l3 < NUM_IPFIX_PROTO_L3; l3++) {
1193             for (l4 = 0; l4 < NUM_IPFIX_PROTO_L4; l4++) {
1194                 if (l3 == IPFIX_PROTO_L3_UNKNOWN &&
1195                     l4 != IPFIX_PROTO_L4_UNKNOWN) {
1196                     continue;
1197                 }
1198                 for (tunnel = 0; tunnel < NUM_IPFIX_PROTO_TUNNEL; tunnel++) {
1199                     /* When the size of the template packet reaches
1200                      * MAX_MESSAGE_LEN(1024), send it out.
1201                      * And then reinitialize the msg to construct a new
1202                      * packet for the following templates.
1203                      */
1204                     if (dp_packet_size(&msg) >= MAX_MESSAGE_LEN) {
1205                         /* Send template message. */
1206                         ipfix_send_template_msg(exporter->collectors,
1207                                                 &msg, set_hdr_offset);
1208
1209                         /* Reinitialize the template msg. */
1210                         ipfix_init_template_msg(msg_stub, export_time_sec,
1211                                                 exporter->seq_number,
1212                                                 obs_domain_id, &msg,
1213                                                 &set_hdr_offset);
1214                     }
1215
1216                     tmpl_hdr_offset = dp_packet_size(&msg);
1217                     tmpl_hdr = dp_packet_put_zeros(&msg, sizeof *tmpl_hdr);
1218                     tmpl_hdr->template_id = htons(
1219                         ipfix_get_template_id(l2, l3, l4, tunnel));
1220                     field_count =
1221                         ipfix_define_template_fields(l2, l3, l4, tunnel, &msg);
1222                     tmpl_hdr = (struct ipfix_template_record_header*)
1223                         ((uint8_t*)dp_packet_data(&msg) + tmpl_hdr_offset);
1224                     tmpl_hdr->field_count = htons(field_count);
1225                 }
1226             }
1227         }
1228     }
1229
1230     /* Send template message. */
1231     ipfix_send_template_msg(exporter->collectors, &msg, set_hdr_offset);
1232
1233     /* XXX: Add Options Template Sets, at least to define a Flow Keys
1234      * Option Template. */
1235
1236 }
1237
1238 static inline uint32_t
1239 ipfix_hash_flow_key(const struct ipfix_flow_key *flow_key, uint32_t basis)
1240 {
1241     uint32_t hash;
1242     hash = hash_int(flow_key->obs_domain_id, basis);
1243     hash = hash_int(flow_key->template_id, hash);
1244     hash = hash_bytes(flow_key->flow_key_msg_part,
1245                       flow_key->flow_key_msg_part_size, hash);
1246     return hash;
1247 }
1248
1249 static bool
1250 ipfix_flow_key_equal(const struct ipfix_flow_key *a,
1251                      const struct ipfix_flow_key *b)
1252 {
1253     /* The template ID determines the flow key size, so not need to
1254      * compare it. */
1255     return (a->obs_domain_id == b->obs_domain_id
1256             && a->template_id == b->template_id
1257             && memcmp(a->flow_key_msg_part, b->flow_key_msg_part,
1258                       a->flow_key_msg_part_size) == 0);
1259 }
1260
1261 static struct ipfix_flow_cache_entry*
1262 ipfix_cache_find_entry(const struct dpif_ipfix_exporter *exporter,
1263                        const struct ipfix_flow_key *flow_key)
1264 {
1265     struct ipfix_flow_cache_entry *entry;
1266
1267     HMAP_FOR_EACH_WITH_HASH (entry, flow_key_map_node,
1268                              ipfix_hash_flow_key(flow_key, 0),
1269                              &exporter->cache_flow_key_map) {
1270         if (ipfix_flow_key_equal(&entry->flow_key, flow_key)) {
1271             return entry;
1272         }
1273     }
1274
1275     return NULL;
1276 }
1277
1278 static bool
1279 ipfix_cache_next_timeout_msec(const struct dpif_ipfix_exporter *exporter,
1280                               long long int *next_timeout_msec)
1281 {
1282     struct ipfix_flow_cache_entry *entry;
1283
1284     LIST_FOR_EACH (entry, cache_flow_start_timestamp_list_node,
1285                    &exporter->cache_flow_start_timestamp_list) {
1286         *next_timeout_msec = entry->flow_start_timestamp_usec / 1000LL
1287             + 1000LL * exporter->cache_active_timeout;
1288         return true;
1289     }
1290
1291     return false;
1292 }
1293
1294 static void
1295 ipfix_cache_aggregate_entries(struct ipfix_flow_cache_entry *from_entry,
1296                               struct ipfix_flow_cache_entry *to_entry)
1297 {
1298     uint64_t *to_start, *to_end, *from_start, *from_end;
1299     uint16_t *to_min_len, *to_max_len, *from_min_len, *from_max_len;
1300
1301     to_start = &to_entry->flow_start_timestamp_usec;
1302     to_end = &to_entry->flow_end_timestamp_usec;
1303     from_start = &from_entry->flow_start_timestamp_usec;
1304     from_end = &from_entry->flow_end_timestamp_usec;
1305
1306     if (*to_start > *from_start) {
1307         *to_start = *from_start;
1308     }
1309     if (*to_end < *from_end) {
1310         *to_end = *from_end;
1311     }
1312
1313     to_entry->packet_delta_count += from_entry->packet_delta_count;
1314     to_entry->layer2_octet_delta_count += from_entry->layer2_octet_delta_count;
1315
1316     to_entry->octet_delta_count += from_entry->octet_delta_count;
1317     to_entry->octet_delta_sum_of_squares +=
1318         from_entry->octet_delta_sum_of_squares;
1319
1320     to_min_len = &to_entry->minimum_ip_total_length;
1321     to_max_len = &to_entry->maximum_ip_total_length;
1322     from_min_len = &from_entry->minimum_ip_total_length;
1323     from_max_len = &from_entry->maximum_ip_total_length;
1324
1325     if (!*to_min_len || (*from_min_len && *to_min_len > *from_min_len)) {
1326         *to_min_len = *from_min_len;
1327     }
1328     if (*to_max_len < *from_max_len) {
1329         *to_max_len = *from_max_len;
1330     }
1331 }
1332
1333 /* Add an entry into a flow cache.  The entry is either aggregated into
1334  * an existing entry with the same flow key and free()d, or it is
1335  * inserted into the cache. */
1336 static void
1337 ipfix_cache_update(struct dpif_ipfix_exporter *exporter,
1338                    struct ipfix_flow_cache_entry *entry)
1339 {
1340     struct ipfix_flow_cache_entry *old_entry;
1341
1342     old_entry = ipfix_cache_find_entry(exporter, &entry->flow_key);
1343
1344     if (old_entry == NULL) {
1345         hmap_insert(&exporter->cache_flow_key_map, &entry->flow_key_map_node,
1346                     ipfix_hash_flow_key(&entry->flow_key, 0));
1347
1348         /* As the latest entry added into the cache, it should
1349          * logically have the highest flow_start_timestamp_usec, so
1350          * append it at the tail. */
1351         list_push_back(&exporter->cache_flow_start_timestamp_list,
1352                        &entry->cache_flow_start_timestamp_list_node);
1353
1354         /* Enforce exporter->cache_max_flows limit. */
1355         if (hmap_count(&exporter->cache_flow_key_map)
1356             > exporter->cache_max_flows) {
1357             dpif_ipfix_cache_expire_now(exporter, false);
1358         }
1359     } else {
1360         ipfix_cache_aggregate_entries(entry, old_entry);
1361         free(entry);
1362     }
1363 }
1364
1365 static void
1366 ipfix_cache_entry_init(struct ipfix_flow_cache_entry *entry,
1367                        const struct dp_packet *packet, const struct flow *flow,
1368                        uint64_t packet_delta_count, uint32_t obs_domain_id,
1369                        uint32_t obs_point_id, odp_port_t output_odp_port,
1370                        const struct dpif_ipfix_port *tunnel_port,
1371                        const struct flow_tnl *tunnel_key)
1372 {
1373     struct ipfix_flow_key *flow_key;
1374     struct dp_packet msg;
1375     enum ipfix_proto_l2 l2;
1376     enum ipfix_proto_l3 l3;
1377     enum ipfix_proto_l4 l4;
1378     enum ipfix_proto_tunnel tunnel = IPFIX_PROTO_NOT_TUNNELED;
1379     uint8_t ethernet_header_length;
1380     uint16_t ethernet_total_length;
1381
1382     flow_key = &entry->flow_key;
1383     dp_packet_use_stub(&msg, flow_key->flow_key_msg_part,
1384                        sizeof flow_key->flow_key_msg_part);
1385
1386     /* Choose the right template ID matching the protocols in the
1387      * sampled packet. */
1388     l2 = (flow->vlan_tci == 0) ? IPFIX_PROTO_L2_ETH : IPFIX_PROTO_L2_VLAN;
1389
1390     switch(ntohs(flow->dl_type)) {
1391     case ETH_TYPE_IP:
1392         l3 = IPFIX_PROTO_L3_IPV4;
1393         switch(flow->nw_proto) {
1394         case IPPROTO_TCP:
1395         case IPPROTO_UDP:
1396         case IPPROTO_SCTP:
1397             l4 = IPFIX_PROTO_L4_TCP_UDP_SCTP;
1398             break;
1399         case IPPROTO_ICMP:
1400             l4 = IPFIX_PROTO_L4_ICMP;
1401             break;
1402         default:
1403             l4 = IPFIX_PROTO_L4_UNKNOWN;
1404         }
1405         break;
1406     case ETH_TYPE_IPV6:
1407         l3 = IPFIX_PROTO_L3_IPV6;
1408         switch(flow->nw_proto) {
1409         case IPPROTO_TCP:
1410         case IPPROTO_UDP:
1411         case IPPROTO_SCTP:
1412             l4 = IPFIX_PROTO_L4_TCP_UDP_SCTP;
1413             break;
1414         case IPPROTO_ICMPV6:
1415             l4 = IPFIX_PROTO_L4_ICMP;
1416             break;
1417         default:
1418             l4 = IPFIX_PROTO_L4_UNKNOWN;
1419         }
1420         break;
1421     default:
1422         l3 = IPFIX_PROTO_L3_UNKNOWN;
1423         l4 = IPFIX_PROTO_L4_UNKNOWN;
1424     }
1425
1426     if (tunnel_port && tunnel_key) {
1427        tunnel = IPFIX_PROTO_TUNNELED;
1428     }
1429
1430     flow_key->obs_domain_id = obs_domain_id;
1431     flow_key->template_id = ipfix_get_template_id(l2, l3, l4, tunnel);
1432
1433     /* The fields defined in the ipfix_data_record_* structs and sent
1434      * below must match exactly the templates defined in
1435      * ipfix_define_template_fields. */
1436
1437     ethernet_header_length = (l2 == IPFIX_PROTO_L2_VLAN)
1438         ? VLAN_ETH_HEADER_LEN : ETH_HEADER_LEN;
1439     ethernet_total_length = dp_packet_size(packet);
1440
1441     /* Common Ethernet entities. */
1442     {
1443         struct ipfix_data_record_flow_key_common *data_common;
1444
1445         data_common = dp_packet_put_zeros(&msg, sizeof *data_common);
1446         data_common->observation_point_id = htonl(obs_point_id);
1447         data_common->flow_direction =
1448             (output_odp_port == ODPP_NONE) ? INGRESS_FLOW : EGRESS_FLOW;
1449         memcpy(data_common->source_mac_address, flow->dl_src,
1450                sizeof flow->dl_src);
1451         memcpy(data_common->destination_mac_address, flow->dl_dst,
1452                sizeof flow->dl_dst);
1453         data_common->ethernet_type = flow->dl_type;
1454         data_common->ethernet_header_length = ethernet_header_length;
1455     }
1456
1457     if (l2 == IPFIX_PROTO_L2_VLAN) {
1458         struct ipfix_data_record_flow_key_vlan *data_vlan;
1459         uint16_t vlan_id = vlan_tci_to_vid(flow->vlan_tci);
1460         uint8_t priority = vlan_tci_to_pcp(flow->vlan_tci);
1461
1462         data_vlan = dp_packet_put_zeros(&msg, sizeof *data_vlan);
1463         data_vlan->vlan_id = htons(vlan_id);
1464         data_vlan->dot1q_vlan_id = htons(vlan_id);
1465         data_vlan->dot1q_priority = priority;
1466     }
1467
1468     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1469         struct ipfix_data_record_flow_key_ip *data_ip;
1470
1471         data_ip = dp_packet_put_zeros(&msg, sizeof *data_ip);
1472         data_ip->ip_version = (l3 == IPFIX_PROTO_L3_IPV4) ? 4 : 6;
1473         data_ip->ip_ttl = flow->nw_ttl;
1474         data_ip->protocol_identifier = flow->nw_proto;
1475         data_ip->ip_diff_serv_code_point = flow->nw_tos >> 2;
1476         data_ip->ip_precedence = flow->nw_tos >> 5;
1477         data_ip->ip_class_of_service = flow->nw_tos;
1478
1479         if (l3 == IPFIX_PROTO_L3_IPV4) {
1480             struct ipfix_data_record_flow_key_ipv4 *data_ipv4;
1481
1482             data_ipv4 = dp_packet_put_zeros(&msg, sizeof *data_ipv4);
1483             data_ipv4->source_ipv4_address = flow->nw_src;
1484             data_ipv4->destination_ipv4_address = flow->nw_dst;
1485         } else {  /* l3 == IPFIX_PROTO_L3_IPV6 */
1486             struct ipfix_data_record_flow_key_ipv6 *data_ipv6;
1487
1488             data_ipv6 = dp_packet_put_zeros(&msg, sizeof *data_ipv6);
1489             memcpy(data_ipv6->source_ipv6_address, &flow->ipv6_src,
1490                    sizeof flow->ipv6_src);
1491             memcpy(data_ipv6->destination_ipv6_address, &flow->ipv6_dst,
1492                    sizeof flow->ipv6_dst);
1493             data_ipv6->flow_label_ipv6 = flow->ipv6_label;
1494         }
1495     }
1496
1497     if (l4 == IPFIX_PROTO_L4_TCP_UDP_SCTP) {
1498         struct ipfix_data_record_flow_key_transport *data_transport;
1499
1500         data_transport = dp_packet_put_zeros(&msg, sizeof *data_transport);
1501         data_transport->source_transport_port = flow->tp_src;
1502         data_transport->destination_transport_port = flow->tp_dst;
1503     } else if (l4 == IPFIX_PROTO_L4_ICMP) {
1504         struct ipfix_data_record_flow_key_icmp *data_icmp;
1505
1506         data_icmp = dp_packet_put_zeros(&msg, sizeof *data_icmp);
1507         data_icmp->icmp_type = ntohs(flow->tp_src) & 0xff;
1508         data_icmp->icmp_code = ntohs(flow->tp_dst) & 0xff;
1509     }
1510
1511     if (tunnel == IPFIX_PROTO_TUNNELED) {
1512         struct ipfix_data_record_flow_key_tunnel *data_tunnel;
1513         const uint8_t *tun_id;
1514
1515         data_tunnel = dp_packet_put_zeros(&msg, sizeof *data_tunnel +
1516                                              tunnel_port->tunnel_key_length);
1517         data_tunnel->tunnel_source_ipv4_address = tunnel_key->ip_src;
1518         data_tunnel->tunnel_destination_ipv4_address = tunnel_key->ip_dst;
1519         /* The tunnel_protocol_identifier is from tunnel_proto array, which
1520          * contains protocol_identifiers of each tunnel type.
1521          * For the tunnel type on the top of IPSec, which uses the protocol
1522          * identifier of the upper tunnel type is used, the tcp_src and tcp_dst
1523          * are decided based on the protocol identifiers.
1524          * E.g:
1525          * The protocol identifier of DPIF_IPFIX_TUNNEL_IPSEC_GRE is IPPROTO_GRE,
1526          * and both tp_src and tp_dst are zero.
1527          */
1528         data_tunnel->tunnel_protocol_identifier =
1529             tunnel_protocol[tunnel_port->tunnel_type];
1530         data_tunnel->tunnel_source_transport_port = tunnel_key->tp_src;
1531         data_tunnel->tunnel_destination_transport_port = tunnel_key->tp_dst;
1532         data_tunnel->tunnel_type = tunnel_port->tunnel_type;
1533         data_tunnel->tunnel_key_length = tunnel_port->tunnel_key_length;
1534         /* tun_id is in network order, and tunnel key is in low bits. */
1535         tun_id = (const uint8_t *) &tunnel_key->tun_id;
1536         memcpy(data_tunnel->tunnel_key,
1537                &tun_id[8 - tunnel_port->tunnel_key_length],
1538                tunnel_port->tunnel_key_length);
1539     }
1540
1541     flow_key->flow_key_msg_part_size = dp_packet_size(&msg);
1542
1543     {
1544         struct timeval now;
1545         uint64_t layer2_octet_delta_count;
1546
1547         /* Calculate the total matched octet count by considering as
1548          * an approximation that all matched packets have the same
1549          * length. */
1550         layer2_octet_delta_count = packet_delta_count * ethernet_total_length;
1551
1552         xgettimeofday(&now);
1553         entry->flow_end_timestamp_usec = now.tv_usec + 1000000LL * now.tv_sec;
1554         entry->flow_start_timestamp_usec = entry->flow_end_timestamp_usec;
1555         entry->packet_delta_count = packet_delta_count;
1556         entry->layer2_octet_delta_count = layer2_octet_delta_count;
1557     }
1558
1559     if (l3 != IPFIX_PROTO_L3_UNKNOWN) {
1560         uint16_t ip_total_length =
1561             ethernet_total_length - ethernet_header_length;
1562         uint64_t octet_delta_count;
1563
1564         /* Calculate the total matched octet count by considering as
1565          * an approximation that all matched packets have the same
1566          * length. */
1567         octet_delta_count = packet_delta_count * ip_total_length;
1568
1569         entry->octet_delta_count = octet_delta_count;
1570         entry->octet_delta_sum_of_squares = octet_delta_count * ip_total_length;
1571         entry->minimum_ip_total_length = ip_total_length;
1572         entry->maximum_ip_total_length = ip_total_length;
1573     } else {
1574         entry->octet_delta_sum_of_squares = 0;
1575         entry->minimum_ip_total_length = 0;
1576         entry->maximum_ip_total_length = 0;
1577     }
1578 }
1579
1580 /* Send each single data record in its own data set, to simplify the
1581  * implementation by avoiding having to group record by template ID
1582  * before sending. */
1583 static void
1584 ipfix_put_data_set(uint32_t export_time_sec,
1585                    struct ipfix_flow_cache_entry *entry,
1586                    enum ipfix_flow_end_reason flow_end_reason,
1587                    struct dp_packet *msg)
1588 {
1589     size_t set_hdr_offset;
1590     struct ipfix_set_header *set_hdr;
1591
1592     set_hdr_offset = dp_packet_size(msg);
1593
1594     /* Put a Data Set. */
1595     set_hdr = dp_packet_put_zeros(msg, sizeof *set_hdr);
1596     set_hdr->set_id = htons(entry->flow_key.template_id);
1597
1598     /* Copy the flow key part of the data record. */
1599
1600     dp_packet_put(msg, entry->flow_key.flow_key_msg_part,
1601                entry->flow_key.flow_key_msg_part_size);
1602
1603     /* Put the non-key part of the data record. */
1604
1605     {
1606         struct ipfix_data_record_aggregated_common *data_aggregated_common;
1607         uint64_t export_time_usec, flow_start_delta_usec, flow_end_delta_usec;
1608
1609         /* Calculate the negative deltas relative to the export time
1610          * in seconds sent in the header, not the exact export
1611          * time. */
1612         export_time_usec = 1000000LL * export_time_sec;
1613         flow_start_delta_usec = export_time_usec
1614             - entry->flow_start_timestamp_usec;
1615         flow_end_delta_usec = export_time_usec
1616             - entry->flow_end_timestamp_usec;
1617
1618         data_aggregated_common = dp_packet_put_zeros(
1619             msg, sizeof *data_aggregated_common);
1620         data_aggregated_common->flow_start_delta_microseconds = htonl(
1621             flow_start_delta_usec);
1622         data_aggregated_common->flow_end_delta_microseconds = htonl(
1623             flow_end_delta_usec);
1624         data_aggregated_common->packet_delta_count = htonll(
1625             entry->packet_delta_count);
1626         data_aggregated_common->layer2_octet_delta_count = htonll(
1627             entry->layer2_octet_delta_count);
1628         data_aggregated_common->flow_end_reason = flow_end_reason;
1629     }
1630
1631     if (entry->octet_delta_sum_of_squares) {  /* IP packet. */
1632         struct ipfix_data_record_aggregated_ip *data_aggregated_ip;
1633
1634         data_aggregated_ip = dp_packet_put_zeros(
1635             msg, sizeof *data_aggregated_ip);
1636         data_aggregated_ip->octet_delta_count = htonll(
1637             entry->octet_delta_count);
1638         data_aggregated_ip->octet_delta_sum_of_squares = htonll(
1639             entry->octet_delta_sum_of_squares);
1640         data_aggregated_ip->minimum_ip_total_length = htonll(
1641             entry->minimum_ip_total_length);
1642         data_aggregated_ip->maximum_ip_total_length = htonll(
1643             entry->maximum_ip_total_length);
1644     }
1645
1646     set_hdr = (struct ipfix_set_header*)((uint8_t*)dp_packet_data(msg) + set_hdr_offset);
1647     set_hdr->length = htons(dp_packet_size(msg) - set_hdr_offset);
1648 }
1649
1650 /* Send an IPFIX message with a single data record. */
1651 static void
1652 ipfix_send_data_msg(struct dpif_ipfix_exporter *exporter,
1653                     uint32_t export_time_sec,
1654                     struct ipfix_flow_cache_entry *entry,
1655                     enum ipfix_flow_end_reason flow_end_reason)
1656 {
1657     uint64_t msg_stub[DIV_ROUND_UP(MAX_MESSAGE_LEN, 8)];
1658     struct dp_packet msg;
1659     dp_packet_use_stub(&msg, msg_stub, sizeof msg_stub);
1660
1661     ipfix_init_header(export_time_sec, exporter->seq_number++,
1662                       entry->flow_key.obs_domain_id, &msg);
1663     ipfix_put_data_set(export_time_sec, entry, flow_end_reason, &msg);
1664     ipfix_send_msg(exporter->collectors, &msg);
1665
1666     dp_packet_uninit(&msg);
1667 }
1668
1669 static void
1670 dpif_ipfix_sample(struct dpif_ipfix_exporter *exporter,
1671                   const struct dp_packet *packet, const struct flow *flow,
1672                   uint64_t packet_delta_count, uint32_t obs_domain_id,
1673                   uint32_t obs_point_id, odp_port_t output_odp_port,
1674                   const struct dpif_ipfix_port *tunnel_port,
1675                   const struct flow_tnl *tunnel_key)
1676 {
1677     struct ipfix_flow_cache_entry *entry;
1678
1679     /* Create a flow cache entry from the sample. */
1680     entry = xmalloc(sizeof *entry);
1681     ipfix_cache_entry_init(entry, packet, flow, packet_delta_count,
1682                            obs_domain_id, obs_point_id,
1683                            output_odp_port, tunnel_port, tunnel_key);
1684     ipfix_cache_update(exporter, entry);
1685 }
1686
1687 void
1688 dpif_ipfix_bridge_sample(struct dpif_ipfix *di, const struct dp_packet *packet,
1689                          const struct flow *flow,
1690                          odp_port_t input_odp_port, odp_port_t output_odp_port,
1691                          const struct flow_tnl *output_tunnel_key)
1692     OVS_EXCLUDED(mutex)
1693 {
1694     uint64_t packet_delta_count;
1695     const struct flow_tnl *tunnel_key = NULL;
1696     struct dpif_ipfix_port * tunnel_port = NULL;
1697
1698     ovs_mutex_lock(&mutex);
1699     /* Use the sampling probability as an approximation of the number
1700      * of matched packets. */
1701     packet_delta_count = UINT32_MAX / di->bridge_exporter.probability;
1702     if (di->bridge_exporter.options->enable_tunnel_sampling) {
1703         if (output_odp_port == ODPP_NONE && flow->tunnel.ip_dst) {
1704             /* Input tunnel. */
1705             tunnel_key = &flow->tunnel;
1706             tunnel_port = dpif_ipfix_find_port(di, input_odp_port);
1707         }
1708         if (output_odp_port != ODPP_NONE && output_tunnel_key) {
1709             /* Output tunnel, output_tunnel_key must be valid. */
1710             tunnel_key = output_tunnel_key;
1711             tunnel_port = dpif_ipfix_find_port(di, output_odp_port);
1712         }
1713     }
1714     dpif_ipfix_sample(&di->bridge_exporter.exporter, packet, flow,
1715                       packet_delta_count,
1716                       di->bridge_exporter.options->obs_domain_id,
1717                       di->bridge_exporter.options->obs_point_id,
1718                       output_odp_port, tunnel_port, tunnel_key);
1719     ovs_mutex_unlock(&mutex);
1720 }
1721
1722 void
1723 dpif_ipfix_flow_sample(struct dpif_ipfix *di, const struct dp_packet *packet,
1724                        const struct flow *flow, uint32_t collector_set_id,
1725                        uint16_t probability, uint32_t obs_domain_id,
1726                        uint32_t obs_point_id) OVS_EXCLUDED(mutex)
1727 {
1728     struct dpif_ipfix_flow_exporter_map_node *node;
1729     /* Use the sampling probability as an approximation of the number
1730      * of matched packets. */
1731     uint64_t packet_delta_count = USHRT_MAX / probability;
1732
1733     ovs_mutex_lock(&mutex);
1734     node = dpif_ipfix_find_flow_exporter_map_node(di, collector_set_id);
1735     if (node) {
1736         dpif_ipfix_sample(&node->exporter.exporter, packet, flow,
1737                           packet_delta_count, obs_domain_id, obs_point_id,
1738                           ODPP_NONE, NULL, NULL);
1739     }
1740     ovs_mutex_unlock(&mutex);
1741 }
1742
1743 static void
1744 dpif_ipfix_cache_expire(struct dpif_ipfix_exporter *exporter,
1745                         bool forced_end, const uint64_t export_time_usec,
1746                         const uint32_t export_time_sec)
1747 {
1748     struct ipfix_flow_cache_entry *entry, *next_entry;
1749     uint64_t max_flow_start_timestamp_usec;
1750     bool template_msg_sent = false;
1751     enum ipfix_flow_end_reason flow_end_reason;
1752
1753     if (list_is_empty(&exporter->cache_flow_start_timestamp_list)) {
1754         return;
1755     }
1756
1757     max_flow_start_timestamp_usec = export_time_usec -
1758         1000000LL * exporter->cache_active_timeout;
1759
1760     LIST_FOR_EACH_SAFE (entry, next_entry, cache_flow_start_timestamp_list_node,
1761                         &exporter->cache_flow_start_timestamp_list) {
1762         if (forced_end) {
1763             flow_end_reason = FORCED_END;
1764         } else if (entry->flow_start_timestamp_usec
1765                    <= max_flow_start_timestamp_usec) {
1766             flow_end_reason = ACTIVE_TIMEOUT;
1767         } else if (hmap_count(&exporter->cache_flow_key_map)
1768                    > exporter->cache_max_flows) {
1769             /* Enforce exporter->cache_max_flows. */
1770             flow_end_reason = LACK_OF_RESOURCES;
1771         } else {
1772             /* Remaining flows haven't expired yet. */
1773             break;
1774         }
1775
1776         list_remove(&entry->cache_flow_start_timestamp_list_node);
1777         hmap_remove(&exporter->cache_flow_key_map,
1778                     &entry->flow_key_map_node);
1779
1780         if (!template_msg_sent
1781             && (exporter->last_template_set_time + IPFIX_TEMPLATE_INTERVAL)
1782                 <= export_time_sec) {
1783             ipfix_send_template_msgs(exporter, export_time_sec,
1784                                      entry->flow_key.obs_domain_id);
1785             exporter->last_template_set_time = export_time_sec;
1786             template_msg_sent = true;
1787         }
1788
1789         /* XXX: Group multiple data records for the same obs domain id
1790          * into the same message. */
1791         ipfix_send_data_msg(exporter, export_time_sec, entry, flow_end_reason);
1792         free(entry);
1793     }
1794 }
1795
1796 static void
1797 get_export_time_now(uint64_t *export_time_usec, uint32_t *export_time_sec)
1798 {
1799     struct timeval export_time;
1800     xgettimeofday(&export_time);
1801
1802     *export_time_usec = export_time.tv_usec + 1000000LL * export_time.tv_sec;
1803
1804     /* The IPFIX start and end deltas are negative deltas relative to
1805      * the export time, so set the export time 1 second off to
1806      * calculate those deltas. */
1807     if (export_time.tv_usec == 0) {
1808         *export_time_sec = export_time.tv_sec;
1809     } else {
1810         *export_time_sec = export_time.tv_sec + 1;
1811     }
1812 }
1813
1814 static void
1815 dpif_ipfix_cache_expire_now(struct dpif_ipfix_exporter *exporter,
1816                             bool forced_end)
1817 {
1818     uint64_t export_time_usec;
1819     uint32_t export_time_sec;
1820
1821     get_export_time_now(&export_time_usec, &export_time_sec);
1822     dpif_ipfix_cache_expire(exporter, forced_end, export_time_usec,
1823                             export_time_sec);
1824 }
1825
1826 void
1827 dpif_ipfix_run(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
1828 {
1829     uint64_t export_time_usec;
1830     uint32_t export_time_sec;
1831     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
1832
1833     ovs_mutex_lock(&mutex);
1834     get_export_time_now(&export_time_usec, &export_time_sec);
1835     if (di->bridge_exporter.probability > 0) {  /* Bridge exporter enabled. */
1836       dpif_ipfix_cache_expire(
1837           &di->bridge_exporter.exporter, false, export_time_usec,
1838           export_time_sec);
1839     }
1840     HMAP_FOR_EACH (flow_exporter_node, node, &di->flow_exporter_map) {
1841         dpif_ipfix_cache_expire(
1842             &flow_exporter_node->exporter.exporter, false, export_time_usec,
1843             export_time_sec);
1844     }
1845     ovs_mutex_unlock(&mutex);
1846 }
1847
1848 void
1849 dpif_ipfix_wait(struct dpif_ipfix *di) OVS_EXCLUDED(mutex)
1850 {
1851     long long int next_timeout_msec = LLONG_MAX;
1852     struct dpif_ipfix_flow_exporter_map_node *flow_exporter_node;
1853
1854     ovs_mutex_lock(&mutex);
1855     if (di->bridge_exporter.probability > 0) {  /* Bridge exporter enabled. */
1856         if (ipfix_cache_next_timeout_msec(
1857                 &di->bridge_exporter.exporter, &next_timeout_msec)) {
1858             poll_timer_wait_until(next_timeout_msec);
1859         }
1860     }
1861     HMAP_FOR_EACH (flow_exporter_node, node, &di->flow_exporter_map) {
1862         if (ipfix_cache_next_timeout_msec(
1863                 &flow_exporter_node->exporter.exporter, &next_timeout_msec)) {
1864             poll_timer_wait_until(next_timeout_msec);
1865         }
1866     }
1867     ovs_mutex_unlock(&mutex);
1868 }