for (INIT_CONTAINER(ITER, (LIST)->prev, MEMBER); \
&(ITER)->MEMBER != (LIST); \
ASSIGN_CONTAINER(ITER, (ITER)->MEMBER.prev, MEMBER))
+#define LIST_FOR_EACH_REVERSE_SAFE(ITER, PREV, MEMBER, LIST) \
+ for (INIT_CONTAINER(ITER, (LIST)->prev, MEMBER); \
+ (&(ITER)->MEMBER != (LIST) \
+ ? INIT_CONTAINER(PREV, (ITER)->MEMBER.prev, MEMBER), 1 \
+ : 0); \
+ (ITER) = (PREV))
#define LIST_FOR_EACH_REVERSE_CONTINUE(ITER, MEMBER, LIST) \
for (ASSIGN_CONTAINER(ITER, (ITER)->MEMBER.prev, MEMBER); \
&(ITER)->MEMBER != (LIST); \
/* If the database has already changed since we started the
* commit, re-evaluate it immediately to avoid missing a change
* for a while. */
+ loop->cur_cfg = loop->next_cfg;
if (ovsdb_idl_get_seqno(loop->idl) != loop->precommit_seqno) {
poll_immediate_wake();
}
break;
case TXN_UNCHANGED:
+ loop->cur_cfg = loop->next_cfg;
+ break;
+
case TXN_ABORTED:
case TXN_NOT_LOCKED:
case TXN_ERROR:
case TXN_UNCOMMITTED:
case TXN_INCOMPLETE:
OVS_NOT_REACHED();
-
}
ovsdb_idl_txn_destroy(txn);
loop->committing_txn = NULL;
-/* Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014, 2015 Nicira, Inc.
+/* Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 Nicira, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
unsigned int precommit_seqno;
struct ovsdb_idl_txn *open_txn;
+
+ /* These members allow a client a simple, stateless way to keep track of
+ * transactions that commit: when a transaction commits successfully,
+ * ovsdb_idl_loop_commit_and_wait() copies 'next_cfg' to 'cur_cfg'. Thus,
+ * the client can set 'next_cfg' to a value that indicates a successful
+ * commit and check 'cur_cfg' on each iteration. */
+ int64_t cur_cfg;
+ int64_t next_cfg;
};
#define OVSDB_IDL_LOOP_INITIALIZER(IDL) { .idl = (IDL) }
-/* Copyright (c) 2015 Nicira, Inc.
+/* Copyright (c) 2015, 2016 Nicira, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
return bridge_mappings ? bridge_mappings : "";
}
-void
+/* Returns this chassis's Chassis record, if it is available and is currently
+ * amenable to a transaction. */
+const struct sbrec_chassis *
chassis_run(struct controller_ctx *ctx, const char *chassis_id)
{
if (!ctx->ovnsb_idl_txn) {
- return;
+ return NULL;
}
const struct ovsrec_open_vswitch *cfg;
cfg = ovsrec_open_vswitch_first(ctx->ovs_idl);
if (!cfg) {
VLOG_INFO("No Open_vSwitch row defined.");
- return;
+ return NULL;
}
encap_type = smap_get(&cfg->external_ids, "ovn-encap-type");
encap_ip = smap_get(&cfg->external_ids, "ovn-encap-ip");
if (!encap_type || !encap_ip) {
VLOG_INFO("Need to specify an encap type and ip");
- return;
+ return NULL;
}
char *tokstr = xstrdup(encap_type);
if (same) {
/* Nothing changed. */
inited = true;
- return;
+ return chassis_rec;
} else if (!inited) {
struct ds cur_encaps = DS_EMPTY_INITIALIZER;
for (int i = 0; i < chassis_rec->n_encaps; i++) {
free(encaps);
inited = true;
+ return chassis_rec;
}
/* Returns true if the database is all cleaned up, false if more work is
-/* Copyright (c) 2015 Nicira, Inc.
+/* Copyright (c) 2015, 2016 Nicira, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
struct ovsrec_bridge;
void chassis_register_ovs_idl(struct ovsdb_idl *);
-void chassis_run(struct controller_ctx *, const char *chassis_id);
+const struct sbrec_chassis *chassis_run(struct controller_ctx *,
+ const char *chassis_id);
bool chassis_cleanup(struct controller_ctx *, const char *chassis_id);
#endif /* ovn/chassis.h */
static void ovn_flow_log(const struct ovn_flow *, const char *action);
static void ovn_flow_destroy(struct ovn_flow *);
-static ovs_be32 queue_msg(struct ofpbuf *);
-static void queue_flow_mod(struct ofputil_flow_mod *);
-
/* OpenFlow connection to the switch. */
static struct rconn *swconn;
-static void queue_group_mod(struct ofputil_group_mod *);
-
/* Last seen sequence number for 'swconn'. When this differs from
* rconn_get_connection_seqno(rconn), 'swconn' has reconnected. */
static unsigned int seqno;
#undef STATE
};
+/* An in-flight update to the switch's flow table.
+ *
+ * When we receive a barrier reply from the switch with the given 'xid', we
+ * know that the switch is caught up to northbound database sequence number
+ * 'nb_cfg' (and make that available to the client via ofctrl_get_cur_cfg(), so
+ * that it can store it into our Chassis record's nb_cfg column). */
+struct ofctrl_flow_update {
+ struct ovs_list list_node; /* In 'flow_updates'. */
+ ovs_be32 xid; /* OpenFlow transaction ID for barrier. */
+ int64_t nb_cfg; /* Northbound database sequence number. */
+};
+
+static struct ofctrl_flow_update *
+ofctrl_flow_update_from_list_node(const struct ovs_list *list_node)
+{
+ return CONTAINER_OF(list_node, struct ofctrl_flow_update, list_node);
+}
+
+/* Currently in-flight updates. */
+static struct ovs_list flow_updates;
+
+/* nb_cfg of latest committed flow update. */
+static int64_t cur_cfg;
+
/* Current state. */
static enum ofctrl_state state;
* S_CLEAR_FLOWS or S_UPDATE_FLOWS, this is really the option we have. */
static enum mf_field_id mff_ovn_geneve;
+static ovs_be32 queue_msg(struct ofpbuf *);
+
static void ovn_flow_table_destroy(void);
+static struct ofpbuf *encode_flow_mod(struct ofputil_flow_mod *);
static void ovn_group_table_clear(struct group_table *group_table,
bool existing);
+static struct ofpbuf *encode_group_mod(const struct ofputil_group_mod *);
static void ofctrl_recv(const struct ofp_header *, enum ofptype);
swconn = rconn_create(5, 0, DSCP_DEFAULT, 1 << OFP13_VERSION);
tx_counter = rconn_packet_counter_create();
hmap_init(&installed_flows);
+ ovs_list_init(&flow_updates);
}
\f
/* S_NEW, for a new connection.
.table_id = OFPTT_ALL,
.command = OFPFC_DELETE,
};
- queue_flow_mod(&fm);
+ queue_msg(encode_flow_mod(&fm));
VLOG_DBG("clearing all flows");
+ /* Send a group_mod to delete all groups. */
struct ofputil_group_mod gm;
memset(&gm, 0, sizeof gm);
gm.command = OFPGC11_DELETE;
gm.group_id = OFPG_ALL;
gm.command_bucket_id = OFPG15_BUCKET_ALL;
ovs_list_init(&gm.buckets);
- queue_group_mod(&gm);
+ queue_msg(encode_group_mod(&gm));
ofputil_bucket_list_destroy(&gm.buckets);
/* Clear installed_flows, to match the state of the switch. */
ovn_group_table_clear(groups, true);
}
+ /* All flow updates are irrelevant now. */
+ struct ofctrl_flow_update *fup, *next;
+ LIST_FOR_EACH_SAFE (fup, next, list_node, &flow_updates) {
+ ovs_list_remove(&fup->list_node);
+ free(fup);
+ }
+
state = S_UPDATE_FLOWS;
}
static void
recv_S_UPDATE_FLOWS(const struct ofp_header *oh, enum ofptype type)
{
- ofctrl_recv(oh, type);
+ if (type == OFPTYPE_BARRIER_REPLY && !ovs_list_is_empty(&flow_updates)) {
+ struct ofctrl_flow_update *fup = ofctrl_flow_update_from_list_node(
+ ovs_list_front(&flow_updates));
+ if (fup->xid == oh->xid) {
+ if (fup->nb_cfg >= cur_cfg) {
+ cur_cfg = fup->nb_cfg;
+ }
+ ovs_list_remove(&fup->list_node);
+ free(fup);
+ }
+ } else {
+ ofctrl_recv(oh, type);
+ }
}
\f
/* Runs the OpenFlow state machine against 'br_int', which is local to the
ovn_flow_table_destroy();
rconn_packet_counter_destroy(tx_counter);
}
+
+int64_t
+ofctrl_get_cur_cfg(void)
+{
+ return cur_cfg;
+}
\f
static ovs_be32
queue_msg(struct ofpbuf *msg)
\f
/* Flow table update. */
-static void
-queue_flow_mod(struct ofputil_flow_mod *fm)
+static struct ofpbuf *
+encode_flow_mod(struct ofputil_flow_mod *fm)
{
fm->buffer_id = UINT32_MAX;
fm->out_port = OFPP_ANY;
fm->out_group = OFPG_ANY;
- queue_msg(ofputil_encode_flow_mod(fm, OFPUTIL_P_OF13_OXM));
+ return ofputil_encode_flow_mod(fm, OFPUTIL_P_OF13_OXM);
}
+static void
+add_flow_mod(struct ofputil_flow_mod *fm, struct ovs_list *msgs)
+{
+ struct ofpbuf *msg = encode_flow_mod(fm);
+ ovs_list_push_back(msgs, &msg->list_node);
+}
\f
/* group_table. */
}
}
+static struct ofpbuf *
+encode_group_mod(const struct ofputil_group_mod *gm)
+{
+ return ofputil_encode_group_mod(OFP13_VERSION, gm);
+}
+
static void
-queue_group_mod(struct ofputil_group_mod *gm)
+add_group_mod(const struct ofputil_group_mod *gm, struct ovs_list *msgs)
{
- queue_msg(ofputil_encode_group_mod(OFP13_VERSION, gm));
+ struct ofpbuf *msg = encode_group_mod(gm);
+ ovs_list_push_back(msgs, &msg->list_node);
}
\f
*
* This should be called after ofctrl_run() within the main loop. */
void
-ofctrl_put(struct group_table *group_table)
+ofctrl_put(struct group_table *group_table, int64_t nb_cfg)
{
if (!groups) {
groups = group_table;
return;
}
+ /* OpenFlow messages to send to the switch to bring it up-to-date. */
+ struct ovs_list msgs = OVS_LIST_INITIALIZER(&msgs);
+
/* Iterate through all the desired groups. If there are new ones,
* add them to the switch. */
struct group_info *desired;
ds_cstr(&group_string),
&usable_protocols);
if (!error) {
- queue_group_mod(&gm);
+ add_group_mod(&gm, &msgs);
} else {
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
VLOG_ERR_RL(&rl, "new group %s %s", error,
.table_id = i->table_id,
.command = OFPFC_DELETE_STRICT,
};
- queue_flow_mod(&fm);
+ add_flow_mod(&fm, &msgs);
ovn_flow_log(i, "removing installed");
hmap_remove(&installed_flows, &i->match_hmap_node);
.ofpacts_len = d->ofpacts_len,
.command = OFPFC_MODIFY_STRICT,
};
- queue_flow_mod(&fm);
+ add_flow_mod(&fm, &msgs);
ovn_flow_log(i, "updating installed");
/* Replace 'i''s actions by 'd''s. */
.ofpacts_len = d->ofpacts_len,
.command = OFPFC_ADD,
};
- queue_flow_mod(&fm);
+ add_flow_mod(&fm, &msgs);
ovn_flow_log(d, "adding installed");
/* Copy 'd' from 'flow_table' to installed_flows. */
ds_cstr(&group_string),
&usable_protocols);
if (!error) {
- queue_group_mod(&gm);
+ add_group_mod(&gm, &msgs);
} else {
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
VLOG_ERR_RL(&rl, "Error deleting group %d: %s",
free(desired);
}
}
+
+ if (!ovs_list_is_empty(&msgs)) {
+ /* Add a barrier to the list of messages. */
+ struct ofpbuf *barrier = ofputil_encode_barrier_request(OFP13_VERSION);
+ const struct ofp_header *oh = barrier->data;
+ ovs_be32 xid = oh->xid;
+ ovs_list_push_back(&msgs, &barrier->list_node);
+
+ /* Queue the messages. */
+ struct ofpbuf *msg;
+ LIST_FOR_EACH_POP (msg, list_node, &msgs) {
+ queue_msg(msg);
+ }
+
+ /* Track the flow update. */
+ struct ofctrl_flow_update *fup, *prev;
+ LIST_FOR_EACH_REVERSE_SAFE (fup, prev, list_node, &flow_updates) {
+ if (nb_cfg < fup->nb_cfg) {
+ /* This ofctrl_flow_update is for a configuration later than
+ * 'nb_cfg'. This should not normally happen, because it means
+ * that 'nb_cfg' in the SB_Global table of the southbound
+ * database decreased, and it should normally be monotonically
+ * increasing. */
+ VLOG_WARN("nb_cfg regressed from %"PRId64" to %"PRId64,
+ fup->nb_cfg, nb_cfg);
+ ovs_list_remove(&fup->list_node);
+ free(fup);
+ } else if (nb_cfg == fup->nb_cfg) {
+ /* This ofctrl_flow_update is for the same configuration as
+ * 'nb_cfg'. Probably, some change to the physical topology
+ * means that we had to revise the OpenFlow flow table even
+ * though the logical topology did not change. Update fp->xid,
+ * so that we don't send a notification that we're up-to-date
+ * until we're really caught up. */
+ VLOG_DBG("advanced xid target for nb_cfg=%"PRId64, nb_cfg);
+ fup->xid = xid;
+ goto done;
+ } else {
+ break;
+ }
+ }
+
+ /* Add a flow update. */
+ fup = xmalloc(sizeof *fup);
+ ovs_list_push_back(&flow_updates, &fup->list_node);
+ fup->xid = xid;
+ fup->nb_cfg = nb_cfg;
+ done:;
+ } else if (!ovs_list_is_empty(&flow_updates)) {
+ /* Getting up-to-date with 'nb_cfg' didn't require any extra flow table
+ * changes, so whenever we get up-to-date with the most recent flow
+ * table update, we're also up-to-date with 'nb_cfg'. */
+ struct ofctrl_flow_update *fup = ofctrl_flow_update_from_list_node(
+ ovs_list_back(&flow_updates));
+ fup->nb_cfg = nb_cfg;
+ } else {
+ /* We were completely up-to-date before and still are. */
+ cur_cfg = nb_cfg;
+ }
}
-/* Copyright (c) 2015 Nicira, Inc.
+/* Copyright (c) 2015, 2016 Nicira, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
/* Interface for OVN main loop. */
void ofctrl_init(void);
enum mf_field_id ofctrl_run(const struct ovsrec_bridge *br_int);
-void ofctrl_put(struct group_table *group_table);
+void ofctrl_put(struct group_table *group_table, int64_t nb_cfg);
void ofctrl_wait(void);
void ofctrl_destroy(void);
+int64_t ofctrl_get_cur_cfg(void);
struct ovn_flow *ofctrl_dup_flow(struct ovn_flow *source);
sset_destroy(&all_users);
}
+static int64_t
+get_nb_cfg(struct ovsdb_idl *idl)
+{
+ const struct sbrec_sb_global *sb = sbrec_sb_global_first(idl);
+ return sb ? sb->nb_cfg : 0;
+}
+
/* Contains "struct local_datapath" nodes whose hash values are the
* tunnel_key of datapaths with at least one local port binding. */
static struct hmap local_datapaths = HMAP_INITIALIZER(&local_datapaths);
char *ovnsb_remote = get_ovnsb_remote(ovs_idl_loop.idl);
struct ovsdb_idl_loop ovnsb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
ovsdb_idl_create(ovnsb_remote, &sbrec_idl_class, true, true));
+ ovsdb_idl_omit_alert(ovnsb_idl_loop.idl, &sbrec_chassis_col_nb_cfg);
/* Track the southbound idl. */
ovsdb_idl_track_add_all(ovnsb_idl_loop.idl);
const struct ovsrec_bridge *br_int = get_br_int(&ctx);
const char *chassis_id = get_chassis_id(ctx.ovs_idl);
+ const struct sbrec_chassis *chassis = NULL;
if (chassis_id) {
- chassis_run(&ctx, chassis_id);
+ chassis = chassis_run(&ctx, chassis_id);
encaps_run(&ctx, br_int, chassis_id);
binding_run(&ctx, br_int, chassis_id, &local_datapaths);
}
br_int, chassis_id, &ct_zones,
&local_datapaths, &patched_datapaths);
- ofctrl_put(&group_table);
+ ofctrl_put(&group_table, get_nb_cfg(ctx.ovnsb_idl));
+ if (ctx.ovnsb_idl_txn) {
+ int64_t cur_cfg = ofctrl_get_cur_cfg();
+ if (cur_cfg && cur_cfg != chassis->nb_cfg) {
+ sbrec_chassis_set_nb_cfg(chassis, cur_cfg);
+ }
+ }
}
sset_destroy(&all_lports);
}
\f
static void
-ovnnb_db_run(struct northd_context *ctx)
+ovnnb_db_run(struct northd_context *ctx, struct ovsdb_idl_loop *sb_loop)
{
- if (!ctx->ovnsb_txn) {
+ if (!ctx->ovnsb_txn || !ovsdb_idl_has_ever_connected(ctx->ovnnb_idl)) {
return;
}
struct hmap datapaths, ports;
ovn_port_destroy(&ports, port);
}
hmap_destroy(&ports);
+
+ /* Copy nb_cfg from northbound to southbound database.
+ *
+ * Also set up to update sb_cfg once our southbound transaction commits. */
+ const struct nbrec_nb_global *nb = nbrec_nb_global_first(ctx->ovnnb_idl);
+ const struct sbrec_sb_global *sb = sbrec_sb_global_first(ctx->ovnsb_idl);
+ if (nb && sb) {
+ sbrec_sb_global_set_nb_cfg(sb, nb->nb_cfg);
+ sb_loop->next_cfg = nb->nb_cfg;
+ }
}
-/*
- * The only change we get notified about is if the 'chassis' column of the
- * 'Port_Binding' table changes. When this column is not empty, it means we
- * need to set the corresponding logical port as 'up' in the northbound DB.
- */
+/* Handle changes to the 'chassis' column of the 'Port_Binding' table. When
+ * this column is not empty, it means we need to set the corresponding logical
+ * port as 'up' in the northbound DB. */
static void
-ovnsb_db_run(struct northd_context *ctx)
+update_logical_port_status(struct northd_context *ctx)
{
- if (!ctx->ovnnb_txn) {
- return;
- }
struct hmap lports_hmap;
const struct sbrec_port_binding *sb;
const struct nbrec_logical_switch_port *nbsp;
}
hmap_destroy(&lports_hmap);
}
-\f
static struct dhcp_opts_map supported_dhcp_opts[] = {
OFFERIP,
hmap_destroy(&dhcp_opts_to_add);
}
+/* Updates the sb_cfg and hv_cfg columns in the northbound NB_Global table. */
+static void
+update_northbound_cfg(struct northd_context *ctx,
+ struct ovsdb_idl_loop *sb_loop)
+{
+ /* Update northbound sb_cfg if appropriate. */
+ const struct nbrec_nb_global *nbg = nbrec_nb_global_first(ctx->ovnnb_idl);
+ int64_t sb_cfg = sb_loop->cur_cfg;
+ if (nbg && sb_cfg && nbg->sb_cfg != sb_cfg) {
+ nbrec_nb_global_set_sb_cfg(nbg, sb_cfg);
+ }
+
+ /* Update northbound hv_cfg if appropriate. */
+ if (nbg) {
+ /* Find minimum nb_cfg among all chassis. */
+ const struct sbrec_chassis *chassis;
+ int64_t hv_cfg = nbg->nb_cfg;
+ SBREC_CHASSIS_FOR_EACH (chassis, ctx->ovnsb_idl) {
+ if (chassis->nb_cfg < hv_cfg) {
+ hv_cfg = chassis->nb_cfg;
+ }
+ }
+
+ /* Update hv_cfg. */
+ if (nbg->hv_cfg != hv_cfg) {
+ nbrec_nb_global_set_hv_cfg(nbg, hv_cfg);
+ }
+ }
+}
+
+/* Handle a fairly small set of changes in the southbound database. */
+static void
+ovnsb_db_run(struct northd_context *ctx, struct ovsdb_idl_loop *sb_loop)
+{
+ if (!ctx->ovnnb_txn || !ovsdb_idl_has_ever_connected(ctx->ovnsb_idl)) {
+ return;
+ }
+
+ update_logical_port_status(ctx);
+ update_northbound_cfg(ctx, sb_loop);
+}
+\f
static char *default_nb_db_;
static const char *
nbrec_init();
sbrec_init();
- /* We want to detect all changes to the ovn-nb db. */
+ /* We want to detect (almost) all changes to the ovn-nb db. */
struct ovsdb_idl_loop ovnnb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
ovsdb_idl_create(ovnnb_db, &nbrec_idl_class, true, true));
+ ovsdb_idl_omit_alert(ovnnb_idl_loop.idl, &nbrec_nb_global_col_sb_cfg);
+ ovsdb_idl_omit_alert(ovnnb_idl_loop.idl, &nbrec_nb_global_col_hv_cfg);
+ /* We want to detect only selected changes to the ovn-sb db. */
struct ovsdb_idl_loop ovnsb_idl_loop = OVSDB_IDL_LOOP_INITIALIZER(
ovsdb_idl_create(ovnsb_db, &sbrec_idl_class, false, true));
+ ovsdb_idl_add_table(ovnsb_idl_loop.idl, &sbrec_table_sb_global);
+ add_column_noalert(ovnsb_idl_loop.idl, &sbrec_sb_global_col_nb_cfg);
+
ovsdb_idl_add_table(ovnsb_idl_loop.idl, &sbrec_table_logical_flow);
add_column_noalert(ovnsb_idl_loop.idl,
&sbrec_logical_flow_col_logical_datapath);
add_column_noalert(ovnsb_idl_loop.idl, &sbrec_address_set_col_name);
add_column_noalert(ovnsb_idl_loop.idl, &sbrec_address_set_col_addresses);
+ ovsdb_idl_add_table(ovnsb_idl_loop.idl, &sbrec_table_chassis);
+ ovsdb_idl_add_column(ovnsb_idl_loop.idl, &sbrec_chassis_col_nb_cfg);
+
/* Main loop. */
exiting = false;
while (!exiting) {
.ovnsb_txn = ovsdb_idl_loop_run(&ovnsb_idl_loop),
};
- ovnnb_db_run(&ctx);
- ovnsb_db_run(&ctx);
+ ovnnb_db_run(&ctx, &ovnsb_idl_loop);
+ ovnsb_db_run(&ctx, &ovnsb_idl_loop);
if (ctx.ovnsb_txn) {
check_and_add_supported_dhcp_opts_to_sb_db(&ctx);
}
+-------------------------------+ +-------------------------------+
</pre>
+ <h2>Information Flow in OVN</h2>
+
+ <p>
+ Configuration data in OVN flows from north to south. The CMS, through its
+ OVN/CMS plugin, passes the logical network configuration to
+ <code>ovn-northd</code> via the northbound database. In turn,
+ <code>ovn-northd</code> compiles the configuration into a lower-level form
+ and passes it to all of the chassis via the southbound database.
+ </p>
+
+ <p>
+ Status information in OVN flows from south to north. OVN currently
+ provides only a few forms of status information. First,
+ <code>ovn-northd</code> populates the <code>up</code> column in the
+ northbound <code>Logical_Switch_Port</code> table: if a logical port's
+ <code>chassis</code> column in the southbound <code>Port_Binding</code>
+ table is nonempty, it sets <code>up</code> to <code>true</code>, otherwise
+ to <code>false</code>. This allows the CMS to detect when a VM's
+ networking has come up.
+ </p>
+
+ <p>
+ Second, OVN provides feedback to the CMS on the realization of its
+ configuration, that is, whether the configuration provided by the CMS has
+ taken effect. This feature requires the CMS to participate in a sequence
+ number protocol, which works the following way:
+ </p>
+
+ <ol>
+ <li>
+ When the CMS updates the configuration in the northbound database, as
+ part of the same transaction, it increments the value of the
+ <code>nb_cfg</code> column in the <code>NB_Global</code> table. (This is
+ only necessary if the CMS wants to know when the configuration has been
+ realized.)
+ </li>
+
+ <li>
+ When <code>ovn-northd</code> updates the southbound database based on a
+ given snapshot of the northbound database, it copies <code>nb_cfg</code>
+ from northbound <code>NB_Global</code> into the southbound database
+ <code>SB_Global</code> table, as part of the same transaction. (Thus, an
+ observer monitoring both databases can determine when the southbound
+ database is caught up with the northbound.)
+ </li>
+
+ <li>
+ After <code>ovn-northd</code> receives confirmation from the southbound
+ database server that its changes have committed, it updates
+ <code>sb_cfg</code> in the northbound <code>NB_Global</code> table to the
+ <code>nb_cfg</code> version that was pushed down. (Thus, the CMS or
+ another observer can determine when the southbound database is caught up
+ without a connection to the southbound database.)
+ </li>
+
+ <li>
+ The <code>ovn-controller</code> process on each chassis receives the
+ updated southbound database, with the updated <code>nb_cfg</code>. This
+ process in turn updates the physical flows installed in the chassis's
+ Open vSwitch instances. When it receives confirmation from Open vSwitch
+ that the physical flows have been updated, it updates <code>nb_cfg</code>
+ in its own <code>Chassis</code> record in the southbound database.
+ </li>
+
+ <li>
+ <code>ovn-northd</code> monitors the <code>nb_cfg</code> column in all of
+ the <code>Chassis</code> records in the southbound database. It keeps
+ track of the minimum value among all the records and copies it into the
+ <code>hv_cfg</code> column in the northbound <code>NB_Global</code>
+ table. (Thus, the CMS or another observer can determine when all of the
+ hypervisors have caught up to the northbound configuration.)
+ </li>
+ </ol>
+
<h2>Chassis Setup</h2>
<p>
{
"name": "OVN_Northbound",
- "version": "5.1.0",
- "cksum": "2201958537 8295",
+ "version": "5.2.0",
+ "cksum": "650844440 8727",
"tables": {
+ "NB_Global": {
+ "columns": {
+ "nb_cfg": {"type": {"key": "integer"}},
+ "sb_cfg": {"type": {"key": "integer"}},
+ "hv_cfg": {"type": {"key": "integer"}},
+ "external_ids": {
+ "type": {"key": "string", "value": "string",
+ "min": 0, "max": "unlimited"}}},
+ "maxRows": 1,
+ "isRoot": true},
"Logical_Switch": {
"columns": {
"name": {"type": "string"},
</dd>
</dl>
+ <table name="NB_Global" title="Northbound configuration">
+ <p>
+ Northbound configuration for an OVN system. This table must have exactly
+ one row.
+ </p>
+
+ <group title="Status">
+ These columns allow a client to track the overall configuration state of
+ the system.
+
+ <column name="nb_cfg">
+ Sequence number for client to increment. When a client modifies any
+ part of the northbound database configuration and wishes to wait for
+ <code>ovn-northd</code> and possibly all of the hypervisors to finish
+ applying the changes, it may increment this sequence number.
+ </column>
+
+ <column name="sb_cfg">
+ Sequence number that <code>ovn-northd</code> sets to the value of <ref
+ column="nb_cfg"/> after it finishes applying the corresponding
+ configuration changes to the <ref db="OVN_Southbound"/> database.
+ </column>
+
+ <column name="hv_cfg">
+ Sequence number that <code>ovn-northd</code> sets to the smallest
+ sequence number of all the chassis in the system, as reported in the
+ <code>Chassis</code> table in the southbound database. Thus, <ref
+ column="hv_cfg"/> equals <ref column="nb_cfg"/> if all chassis are
+ caught up with the northbound configuration (which may never happen, if
+ any chassis is down). This value can regress, if a chassis was removed
+ from the system and rejoins before catching up.
+ </column>
+ </group>
+
+ <group title="Common Columns">
+ <column name="external_ids">
+ See <em>External IDs</em> at the beginning of this document.
+ </column>
+ </group>
+ </table>
+
<table name="Logical_Switch" title="L2 logical switch">
<p>
Each row represents one L2 logical switch.
{
"name": "OVN_Southbound",
- "version": "1.6.0",
- "cksum": "1715817174 6541",
+ "version": "1.7.0",
+ "cksum": "3677179333 6917",
"tables": {
+ "SB_Global": {
+ "columns": {
+ "nb_cfg": {"type": {"key": "integer"}},
+ "external_ids": {
+ "type": {"key": "string", "value": "string",
+ "min": 0, "max": "unlimited"}}},
+ "maxRows": 1,
+ "isRoot": true},
"Chassis": {
"columns": {
"name": {"type": "string"},
"vtep_logical_switches" : {"type": {"key": "string",
"min": 0,
"max": "unlimited"}},
+ "nb_cfg": {"type": {"key": "integer"}},
"external_ids": {
"type": {"key": "string", "value": "string",
"min": 0, "max": "unlimited"}}},
</dd>
</dl>
+ <table name="SB_Global" title="Southbound configuration">
+ <p>
+ Southbound configuration for an OVN system. This table must have exactly
+ one row.
+ </p>
+
+ <group title="Status">
+ This column allow a client to track the overall configuration state of
+ the system.
+
+ <column name="nb_cfg">
+ Sequence number for the configuration. When a CMS or
+ <code>ovn-nbctl</code> updates the northbound database, it increments
+ the <code>nb_cfg</code> column in the <code>NB_Global</code> table in
+ the northbound database. In turn, when <code>ovn-northd</code> updates
+ the southbound database to bring it up to date with these changes, it
+ updates this column to the same value.
+ </column>
+ </group>
+
+ <group title="Common Columns">
+ <column name="external_ids">
+ See <em>External IDs</em> at the beginning of this document.
+ </column>
+ </group>
+ </table>
+
<table name="Chassis" title="Physical Network Hypervisor and Gateway Information">
<p>
Each row in this table represents a hypervisor or gateway (a chassis) in
ovn-controller-vtep will leave this column empty.
</column>
+ <column name="nb_cfg">
+ Sequence number for the configuration. When <code>ovn-controller</code>
+ updates the configuration of a chassis from the contents of the
+ southbound database, it copies <ref table="SB_Global" column="nb_cfg"/>
+ from the <ref table="SB_Global"/> table into this column.
+ </column>
+
<column name="external_ids" key="ovn-bridge-mappings">
<code>ovn-controller</code> populates this key with the set of bridge
mappings it has been configured to use. Other applications should treat
<h1>General Commands</h1>
<dl>
+ <dt><code>init</code></dt>
+ <dd>
+ Initializes the database, if it is empty. If the database has already
+ been initialized, this command has no effect.
+ </dd>
+
<dt><code>show [<var>switch</var> | <var>router</var>]</code></dt>
<dd>
Prints a brief overview of the database contents. If
<h1>Options</h1>
<dl>
+ <dt><code>--no-wait</code> | <code>--wait=none</code></dt>
+ <dt><code>--wait=sb</code></dt>
+ <dt><code>--wait=hv</code></dt>
+
+ <dd>
+ <p>
+ These options control whether and how <code>ovn-nbctl</code> waits
+ for the OVN system to become up-to-date with changes made in an
+ <code>ovn-nbctl</code> invocation.
+ </p>
+
+ <p>
+ By default, or if <code>--no-wait</code> or <code>--wait=none</code>,
+ <code>ovn-nbctl</code> exits immediately after confirming that
+ changes have been committed to the northbound database, without
+ waiting.
+ </p>
+
+ <p>
+ With <code>--wait=sb</code>, before <code>ovn-nbctl</code> exits, it
+ waits for <code>ovn-northd</code> to bring the southbound database
+ up-to-date with the northbound database updates.
+ </p>
+
+ <p>
+ With <code>--wait=hv</code>, before <code>ovn-nbctl</code> exits, it
+ additionally waits for all OVN chassis (hypervisors and gateways) to
+ become up-to-date with the northbound database updates. (This can
+ become an indefinite wait if any chassis is malfunctioning.)
+ </p>
+ </dd>
+
<dt><code>--db</code> <var>database</var></dt>
<dd>
The OVSDB database remote to contact. If the <env>OVN_NB_DB</env>
/* --dry-run: Do not commit any changes. */
static bool dry_run;
+/* --wait=TYPE: Wait for configuration change to take effect? */
+enum nbctl_wait_type {
+ NBCTL_WAIT_NONE, /* Do not wait. */
+ NBCTL_WAIT_SB, /* Wait for southbound database updates. */
+ NBCTL_WAIT_HV /* Wait for hypervisors to catch up. */
+};
+static enum nbctl_wait_type wait_type = NBCTL_WAIT_NONE;
+
/* --timeout: Time to wait for a connection to 'db'. */
static int timeout;
enum {
OPT_DB = UCHAR_MAX + 1,
OPT_NO_SYSLOG,
+ OPT_NO_WAIT,
+ OPT_WAIT,
OPT_DRY_RUN,
OPT_ONELINE,
OPT_LOCAL,
static const struct option global_long_options[] = {
{"db", required_argument, NULL, OPT_DB},
{"no-syslog", no_argument, NULL, OPT_NO_SYSLOG},
+ {"no-wait", no_argument, NULL, OPT_NO_WAIT},
+ {"wait", required_argument, NULL, OPT_WAIT},
{"dry-run", no_argument, NULL, OPT_DRY_RUN},
{"oneline", no_argument, NULL, OPT_ONELINE},
{"timeout", required_argument, NULL, 't'},
vlog_set_levels(&this_module, VLF_SYSLOG, VLL_WARN);
break;
+ case OPT_NO_WAIT:
+ wait_type = NBCTL_WAIT_NONE;
+ break;
+
+ case OPT_WAIT:
+ if (!strcmp(optarg, "none")) {
+ wait_type = NBCTL_WAIT_NONE;
+ } else if (!strcmp(optarg, "sb")) {
+ wait_type = NBCTL_WAIT_SB;
+ } else if (!strcmp(optarg, "hv")) {
+ wait_type = NBCTL_WAIT_HV;
+ } else {
+ ctl_fatal("argument to --wait must be "
+ "\"none\", \"sb\", or \"hv\"");
+ }
+ break;
+
case OPT_DRY_RUN:
dry_run = true;
break;
usage: %s [OPTIONS] COMMAND [ARG...]\n\
\n\
General commands:\n\
+ init initialize the database\n\
show print overview of database contents\n\
show SWITCH print overview of database contents for SWITCH\n\
show ROUTER print overview of database contents for ROUTER\n\
Options:\n\
--db=DATABASE connect to DATABASE\n\
(default: %s)\n\
+ --no-wait, --wait=none do not wait for OVN reconfiguration (default)\n\
+ --wait=sb wait for southbound database update\n\
+ --wait=hv wait for all chassis to catch up\n\
-t, --timeout=SECS wait at most SECS seconds\n\
--dry-run do not commit changes to database\n\
--oneline print exactly one line of output per command\n",
}
}
+static void
+nbctl_init(struct ctl_context *ctx OVS_UNUSED)
+{
+}
+
static void
nbctl_show(struct ctl_context *ctx)
{
}
static const struct ctl_table_class tables[] = {
+ {&nbrec_table_nb_global,
+ {{&nbrec_table_nb_global, NULL, NULL},
+ {NULL, NULL, NULL}}},
+
{&nbrec_table_logical_switch,
{{&nbrec_table_logical_switch, &nbrec_logical_switch_col_name, NULL},
{NULL, NULL, NULL}}},
run_prerequisites(struct ctl_command *commands, size_t n_commands,
struct ovsdb_idl *idl)
{
- struct ctl_command *c;
+ ovsdb_idl_add_table(idl, &nbrec_table_nb_global);
+ if (wait_type == NBCTL_WAIT_SB) {
+ ovsdb_idl_add_column(idl, &nbrec_nb_global_col_sb_cfg);
+ } else if (wait_type == NBCTL_WAIT_HV) {
+ ovsdb_idl_add_column(idl, &nbrec_nb_global_col_hv_cfg);
+ }
- for (c = commands; c < &commands[n_commands]; c++) {
+ for (struct ctl_command *c = commands; c < &commands[n_commands]; c++) {
if (c->syntax->prerequisites) {
struct ctl_context ctx;
struct ctl_context ctx;
struct ctl_command *c;
struct shash_node *node;
+ int64_t next_cfg = 0;
char *error = NULL;
txn = the_idl_txn = ovsdb_idl_txn_create(idl);
ovsdb_idl_txn_add_comment(txn, "ovs-nbctl: %s", args);
+ const struct nbrec_nb_global *nb = nbrec_nb_global_first(idl);
+ if (!nb) {
+ /* XXX add verification that table is empty */
+ nb = nbrec_nb_global_insert(txn);
+ }
+
+ if (wait_type != NBCTL_WAIT_NONE) {
+ ovsdb_idl_txn_increment(txn, &nb->header_,
+ &nbrec_nb_global_col_nb_cfg);
+ }
+
symtab = ovsdb_symbol_table_create();
for (c = commands; c < &commands[n_commands]; c++) {
ds_init(&c->output);
}
status = ovsdb_idl_txn_commit_block(txn);
+ if (wait_type != NBCTL_WAIT_NONE && status == TXN_SUCCESS) {
+ next_cfg = ovsdb_idl_txn_get_increment_new_value(txn);
+ }
if (status == TXN_UNCHANGED || status == TXN_SUCCESS) {
for (c = commands; c < &commands[n_commands]; c++) {
if (c->syntax->postprocess) {
shash_destroy_free_data(&c->options);
}
free(commands);
+
+ if (wait_type != NBCTL_WAIT_NONE && status != TXN_UNCHANGED) {
+ ovsdb_idl_enable_reconnect(idl);
+ for (;;) {
+ ovsdb_idl_run(idl);
+ NBREC_NB_GLOBAL_FOR_EACH (nb, idl) {
+ int64_t cur_cfg = (wait_type == NBCTL_WAIT_SB
+ ? nb->sb_cfg
+ : nb->hv_cfg);
+ if (cur_cfg >= next_cfg) {
+ goto done;
+ }
+ }
+ ovsdb_idl_wait(idl);
+ poll_block();
+ }
+ done: ;
+ }
+
ovsdb_idl_txn_destroy(txn);
ovsdb_idl_destroy(idl);
}
static const struct ctl_command_syntax nbctl_commands[] = {
+ { "init", 0, 0, "", NULL, nbctl_init, NULL, "", RW },
{ "show", 0, 1, "[SWITCH]", NULL, nbctl_show, NULL, "", RO },
/* logical switch commands. */
.SS "OVN_Southbound Commands"
These commands work with an \fBOVN_Southbound\fR database as a whole.
.
+.IP "\fBinit\fR"
+Initializes the database, if it is empty. If the database has already
+been initialized, this command has no effect.
+.
.IP "\fBshow\fR"
Prints a brief overview of the database contents.
.
{NULL, NULL, {NULL, NULL, NULL}, {NULL, NULL, NULL}},
};
+static void
+sbctl_init(struct ctl_context *ctx OVS_UNUSED)
+{
+}
+
static void
cmd_chassis_add(struct ctl_context *ctx)
{
\f
static const struct ctl_table_class tables[] = {
+ {&sbrec_table_sb_global,
+ {{&sbrec_table_sb_global, NULL, NULL},
+ {NULL, NULL, NULL}}},
+
{&sbrec_table_chassis,
{{&sbrec_table_chassis, &sbrec_chassis_col_name, NULL},
{NULL, NULL, NULL}}},
run_prerequisites(struct ctl_command *commands, size_t n_commands,
struct ovsdb_idl *idl)
{
- struct ctl_command *c;
+ ovsdb_idl_add_table(idl, &sbrec_table_sb_global);
- for (c = commands; c < &commands[n_commands]; c++) {
+ for (struct ctl_command *c = commands; c < &commands[n_commands]; c++) {
if (c->syntax->prerequisites) {
struct sbctl_context sbctl_ctx;
ovsdb_idl_txn_add_comment(txn, "ovs-sbctl: %s", args);
+ const struct sbrec_sb_global *sb = sbrec_sb_global_first(idl);
+ if (!sb) {
+ /* XXX add verification that table is empty */
+ sb = sbrec_sb_global_insert(txn);
+ }
+
symtab = ovsdb_symbol_table_create();
for (c = commands; c < &commands[n_commands]; c++) {
ds_init(&c->output);
}
static const struct ctl_command_syntax sbctl_commands[] = {
+ { "init", 0, 0, "", NULL, sbctl_init, NULL, "", RW },
+
/* Chassis commands. */
{"chassis-add", 3, 3, "CHASSIS ENCAP-TYPE ENCAP-IP", pre_get_info,
cmd_chassis_add, NULL, "--may-exist", RW},
# ovn-sbctl and ovn-nbctl use them by default, and starts ovn-northd running
# against them.
ovn_start () {
- ovn_init_db ovn-sb
- ovn_init_db ovn-nb
+ ovn_init_db ovn-sb; ovn-sbctl init
+ ovn_init_db ovn-nb; ovn-nbctl init
echo "starting ovn-northd"
mkdir "$ovs_base"/northd
dnl Start ovsdb-server.
AT_CHECK([ovsdb-server --detach --no-chdir --pidfile --log-file --remote=punix:$OVS_RUNDIR/ovnnb_db.sock ovn-nb.db], [0], [], [stderr])
on_exit "kill `cat ovsdb-server.pid`"
+ AT_CHECK([ovn-nbctl init])
AT_CHECK([[sed < stderr '
/vlog|INFO|opened log file/d
/ovsdb_server|INFO|ovsdb-server (Open vSwitch)/d']])
#! /bin/sh
#
-# Copyright (c) 2013, 2015 Nicira, Inc.
+# Copyright (c) 2013, 2015, 2016 Nicira, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
ovs-vsctl set open . external-ids:ovn-encap-type=geneve
ovs-vsctl set open . external-ids:ovn-encap-ip=127.0.0.1
+ ovn-nbctl init
+ ovn-sbctl init
+
rungdb $gdb_ovn_northd $gdb_ovn_northd_ex ovn-northd --detach \
--no-chdir --pidfile -vconsole:off --log-file \
--ovnsb-db=unix:"$sandbox"/ovnsb_db.sock \
#! /usr/bin/env bash
#
-# Copyright (c) 2013, 2015 Nicira, Inc.
+# Copyright (c) 2013, 2015, 2016 Nicira, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
OVN_NB_DB=unix:$sim_base/ovn-nb/ovn-nb.sock; export OVN_NB_DB
OVN_SB_DB=unix:$sim_base/ovn-sb/ovn-sb.sock; export OVN_SB_DB
+ ovn-nbctl init
+ ovn-sbctl init
+
mkdir "$sim_base"/northd
as northd ovn-northd $daemon_opts \
--ovnnb-db="$OVN_NB_DB" \
projects / cascardo / ovs.git / commitdiff