+static struct udpif_key *
+ukey_create_from_upcall(struct upcall *upcall, struct flow_wildcards *wc)
+{
+ struct odputil_keybuf keystub, maskstub;
+ struct ofpbuf keybuf, maskbuf;
+ bool megaflow;
+ struct odp_flow_key_parms odp_parms = {
+ .flow = upcall->flow,
+ .mask = &wc->masks,
+ };
+
+ odp_parms.support = ofproto_dpif_get_support(upcall->ofproto)->odp;
+ if (upcall->key_len) {
+ ofpbuf_use_const(&keybuf, upcall->key, upcall->key_len);
+ } else {
+ /* dpif-netdev doesn't provide a netlink-formatted flow key in the
+ * upcall, so convert the upcall's flow here. */
+ ofpbuf_use_stack(&keybuf, &keystub, sizeof keystub);
+ odp_parms.odp_in_port = upcall->flow->in_port.odp_port;
+ odp_flow_key_from_flow(&odp_parms, &keybuf);
+ }
+
+ atomic_read_relaxed(&enable_megaflows, &megaflow);
+ ofpbuf_use_stack(&maskbuf, &maskstub, sizeof maskstub);
+ if (megaflow) {
+ odp_parms.odp_in_port = ODPP_NONE;
+ odp_parms.key_buf = &keybuf;
+
+ odp_flow_key_from_mask(&odp_parms, &maskbuf);
+ }
+
+ return ukey_create__(keybuf.data, keybuf.size, maskbuf.data, maskbuf.size,
+ true, upcall->ufid, upcall->pmd_id,
+ &upcall->put_actions, upcall->dump_seq,
+ upcall->reval_seq, 0,
+ upcall->have_recirc_ref ? upcall->recirc->id : 0,
+ &upcall->xout);
+}
+
+static int
+ukey_create_from_dpif_flow(const struct udpif *udpif,
+ const struct dpif_flow *flow,
+ struct udpif_key **ukey)
+{
+ struct dpif_flow full_flow;
+ struct ofpbuf actions;
+ uint64_t dump_seq, reval_seq;
+ uint64_t stub[DPIF_FLOW_BUFSIZE / 8];
+ const struct nlattr *a;
+ unsigned int left;
+
+ if (!flow->key_len || !flow->actions_len) {
+ struct ofpbuf buf;
+ int err;
+
+ /* If the key or actions were not provided by the datapath, fetch the
+ * full flow. */
+ ofpbuf_use_stack(&buf, &stub, sizeof stub);
+ err = dpif_flow_get(udpif->dpif, NULL, 0, &flow->ufid,
+ flow->pmd_id, &buf, &full_flow);
+ if (err) {
+ return err;
+ }
+ flow = &full_flow;
+ }
+
+ /* Check the flow actions for recirculation action. As recirculation
+ * relies on OVS userspace internal state, we need to delete all old
+ * datapath flows with either a non-zero recirc_id in the key, or any
+ * recirculation actions upon OVS restart. */
+ NL_ATTR_FOR_EACH_UNSAFE (a, left, flow->key, flow->key_len) {
+ if (nl_attr_type(a) == OVS_KEY_ATTR_RECIRC_ID
+ && nl_attr_get_u32(a) != 0) {
+ return EINVAL;
+ }
+ }
+ NL_ATTR_FOR_EACH_UNSAFE (a, left, flow->actions, flow->actions_len) {
+ if (nl_attr_type(a) == OVS_ACTION_ATTR_RECIRC) {
+ return EINVAL;
+ }
+ }
+
+ dump_seq = seq_read(udpif->dump_seq);
+ reval_seq = seq_read(udpif->reval_seq);
+ ofpbuf_use_const(&actions, &flow->actions, flow->actions_len);
+ *ukey = ukey_create__(flow->key, flow->key_len,
+ flow->mask, flow->mask_len, flow->ufid_present,
+ &flow->ufid, flow->pmd_id, &actions, dump_seq,
+ reval_seq, flow->stats.used, 0, NULL);
+
+ return 0;
+}
+
+/* Attempts to insert a ukey into the shared ukey maps.
+ *
+ * On success, returns true, installs the ukey and returns it in a locked
+ * state. Otherwise, returns false. */
+static bool
+ukey_install_start(struct udpif *udpif, struct udpif_key *new_ukey)
+ OVS_TRY_LOCK(true, new_ukey->mutex)
+{
+ struct umap *umap;
+ struct udpif_key *old_ukey;
+ uint32_t idx;
+ bool locked = false;
+
+ idx = new_ukey->hash % N_UMAPS;
+ umap = &udpif->ukeys[idx];
+ ovs_mutex_lock(&umap->mutex);
+ old_ukey = ukey_lookup(udpif, &new_ukey->ufid);
+ if (old_ukey) {
+ /* Uncommon case: A ukey is already installed with the same UFID. */
+ if (old_ukey->key_len == new_ukey->key_len
+ && !memcmp(old_ukey->key, new_ukey->key, new_ukey->key_len)) {
+ COVERAGE_INC(handler_duplicate_upcall);
+ } else {
+ struct ds ds = DS_EMPTY_INITIALIZER;
+
+ odp_format_ufid(&old_ukey->ufid, &ds);
+ ds_put_cstr(&ds, " ");
+ odp_flow_key_format(old_ukey->key, old_ukey->key_len, &ds);
+ ds_put_cstr(&ds, "\n");
+ odp_format_ufid(&new_ukey->ufid, &ds);
+ ds_put_cstr(&ds, " ");
+ odp_flow_key_format(new_ukey->key, new_ukey->key_len, &ds);
+
+ VLOG_WARN_RL(&rl, "Conflicting ukey for flows:\n%s", ds_cstr(&ds));
+ ds_destroy(&ds);
+ }
+ } else {
+ ovs_mutex_lock(&new_ukey->mutex);
+ cmap_insert(&umap->cmap, &new_ukey->cmap_node, new_ukey->hash);
+ locked = true;
+ }
+ ovs_mutex_unlock(&umap->mutex);
+
+ return locked;
+}
+
+static void
+ukey_install_finish__(struct udpif_key *ukey) OVS_REQUIRES(ukey->mutex)
+{
+ ukey->flow_exists = true;
+}
+
+static bool
+ukey_install_finish(struct udpif_key *ukey, int error)
+ OVS_RELEASES(ukey->mutex)
+{
+ if (!error) {
+ ukey_install_finish__(ukey);
+ }
+ ovs_mutex_unlock(&ukey->mutex);
+
+ return !error;
+}
+
+static bool
+ukey_install(struct udpif *udpif, struct udpif_key *ukey)
+{
+ /* The usual way to keep 'ukey->flow_exists' in sync with the datapath is
+ * to call ukey_install_start(), install the corresponding datapath flow,
+ * then call ukey_install_finish(). The netdev interface using upcall_cb()
+ * doesn't provide a function to separately finish the flow installation,
+ * so we perform the operations together here.
+ *
+ * This is fine currently, as revalidator threads will only delete this
+ * ukey during revalidator_sweep() and only if the dump_seq is mismatched.
+ * It is unlikely for a revalidator thread to advance dump_seq and reach
+ * the next GC phase between ukey creation and flow installation. */
+ return ukey_install_start(udpif, ukey) && ukey_install_finish(ukey, 0);
+}
+
+/* Searches for a ukey in 'udpif->ukeys' that matches 'flow' and attempts to
+ * lock the ukey. If the ukey does not exist, create it.
+ *
+ * Returns 0 on success, setting *result to the matching ukey and returning it
+ * in a locked state. Otherwise, returns an errno and clears *result. EBUSY
+ * indicates that another thread is handling this flow. Other errors indicate
+ * an unexpected condition creating a new ukey.
+ *
+ * *error is an output parameter provided to appease the threadsafety analyser,
+ * and its value matches the return value. */
+static int
+ukey_acquire(struct udpif *udpif, const struct dpif_flow *flow,
+ struct udpif_key **result, int *error)
+ OVS_TRY_LOCK(0, (*result)->mutex)
+{
+ struct udpif_key *ukey;
+ int retval;
+
+ ukey = ukey_lookup(udpif, &flow->ufid);
+ if (ukey) {
+ retval = ovs_mutex_trylock(&ukey->mutex);
+ } else {
+ /* Usually we try to avoid installing flows from revalidator threads,
+ * because locking on a umap may cause handler threads to block.
+ * However there are certain cases, like when ovs-vswitchd is
+ * restarted, where it is desirable to handle flows that exist in the
+ * datapath gracefully (ie, don't just clear the datapath). */
+ bool install;
+
+ retval = ukey_create_from_dpif_flow(udpif, flow, &ukey);
+ if (retval) {
+ goto done;
+ }
+ install = ukey_install_start(udpif, ukey);
+ if (install) {
+ ukey_install_finish__(ukey);
+ retval = 0;
+ } else {
+ ukey_delete__(ukey);
+ retval = EBUSY;
+ }
+ }
+
+done:
+ *error = retval;
+ if (retval) {
+ *result = NULL;
+ } else {
+ *result = ukey;
+ }
+ return retval;
+}
+