ovn: Apply ACL changes to existing connections.
[cascardo/ovs.git] / ovn / northd / ovn-northd.c
index 26a9526..aa7b65f 100644 (file)
@@ -175,6 +175,20 @@ ovn_stage_to_str(enum ovn_stage stage)
         default: return "<unknown>";
     }
 }
+
+/* Returns the type of the datapath to which a flow with the given 'stage' may
+ * be added. */
+static enum ovn_datapath_type
+ovn_stage_to_datapath_type(enum ovn_stage stage)
+{
+    switch (stage) {
+#define PIPELINE_STAGE(DP_TYPE, PIPELINE, STAGE, TABLE, NAME)       \
+        case S_##DP_TYPE##_##PIPELINE##_##STAGE: return DP_##DP_TYPE;
+    PIPELINE_STAGES
+#undef PIPELINE_STAGE
+    default: OVS_NOT_REACHED();
+    }
+}
 \f
 static void
 usage(void)
@@ -303,6 +317,13 @@ ovn_datapath_destroy(struct hmap *datapaths, struct ovn_datapath *od)
     }
 }
 
+/* Returns 'od''s datapath type. */
+static enum ovn_datapath_type
+ovn_datapath_get_type(const struct ovn_datapath *od)
+{
+    return od->nbs ? DP_SWITCH : DP_ROUTER;
+}
+
 static struct ovn_datapath *
 ovn_datapath_find(struct hmap *datapaths, const struct uuid *uuid)
 {
@@ -494,16 +515,8 @@ struct ovn_port {
     /* Logical router port data. */
     const struct nbrec_logical_router_port *nbr; /* May be NULL. */
 
-    char *ip_s;                 /* "192.168.10.123" */
-    char *network_s;            /* "192.168.10.0" */
-    char *bcast_s;              /* "192.168.10.255" */
-    int plen;                   /* CIDR prefix: 24 */
+    struct lport_addresses lrp_networks;
 
-    ovs_be32 ip;                /* 192.168.10.123 */
-    ovs_be32 mask;              /* 255.255.255.0 */
-    ovs_be32 network;           /* 192.168.10.255 */
-
-    struct eth_addr mac;
     struct ovn_port *peer;
 
     struct ovn_datapath *od;
@@ -550,9 +563,7 @@ ovn_port_destroy(struct hmap *ports, struct ovn_port *port)
         }
         free(port->ps_addrs);
 
-        free(port->bcast_s);
-        free(port->network_s);
-        free(port->ip_s);
+        destroy_lport_addresses(&port->lrp_networks);
         free(port->json_key);
         free(port->key);
         free(port);
@@ -661,24 +672,17 @@ join_logical_ports(struct northd_context *ctx,
             }
         } else {
             for (size_t i = 0; i < od->nbr->n_ports; i++) {
-                const struct nbrec_logical_router_port *nbr
-                    = od->nbr->ports[i];
+                const struct nbrec_logical_router_port *nbr = od->nbr->ports[i];
 
-                struct eth_addr mac;
-                if (!eth_addr_from_string(nbr->mac, &mac)) {
+                struct lport_addresses lrp_networks;
+                if (!extract_lrp_networks(nbr, &lrp_networks)) {
                     static struct vlog_rate_limit rl
                         = VLOG_RATE_LIMIT_INIT(5, 1);
                     VLOG_WARN_RL(&rl, "bad 'mac' %s", nbr->mac);
                     continue;
                 }
 
-                ovs_be32 ip, mask;
-                char *error = ip_parse_masked(nbr->network, &ip, &mask);
-                if (error || mask == OVS_BE32_MAX || !ip_is_cidr(mask)) {
-                    static struct vlog_rate_limit rl
-                        = VLOG_RATE_LIMIT_INIT(5, 1);
-                    VLOG_WARN_RL(&rl, "bad 'network' %s", nbr->network);
-                    free(error);
+                if (!lrp_networks.n_ipv4_addrs && !lrp_networks.n_ipv6_addrs) {
                     continue;
                 }
 
@@ -694,21 +698,17 @@ join_logical_ports(struct northd_context *ctx,
                     op->nbr = nbr;
                     ovs_list_remove(&op->list);
                     ovs_list_push_back(both, &op->list);
+
+                    /* This port exists but should not have been
+                     * initialized fully. */
+                    ovs_assert(!op->lrp_networks.n_ipv4_addrs
+                               && !op->lrp_networks.n_ipv6_addrs);
                 } else {
                     op = ovn_port_create(ports, nbr->name, NULL, nbr, NULL);
                     ovs_list_push_back(nb_only, &op->list);
                 }
 
-                op->ip = ip;
-                op->mask = mask;
-                op->network = ip & mask;
-                op->plen = ip_count_cidr_bits(mask);
-
-                op->ip_s = xasprintf(IP_FMT, IP_ARGS(ip));
-                op->network_s = xasprintf(IP_FMT, IP_ARGS(op->network));
-                op->bcast_s = xasprintf(IP_FMT, IP_ARGS(ip | ~mask));
-
-                op->mac = mac;
+                op->lrp_networks = lrp_networks;
                 op->od = od;
             }
         }
@@ -1006,6 +1006,8 @@ ovn_lflow_add(struct hmap *lflow_map, struct ovn_datapath *od,
               enum ovn_stage stage, uint16_t priority,
               const char *match, const char *actions)
 {
+    ovs_assert(ovn_stage_to_datapath_type(stage) == ovn_datapath_get_type(od));
+
     struct ovn_lflow *lflow = xmalloc(sizeof *lflow);
     ovn_lflow_init(lflow, od, stage, priority,
                    xstrdup(match), xstrdup(actions));
@@ -1165,18 +1167,18 @@ build_port_security_nd(struct ovn_port *op, struct hmap *lflows)
 
             if (ps->n_ipv4_addrs) {
                 ds_put_cstr(&match, " && arp.spa == {");
-                for (size_t i = 0; i < ps->n_ipv4_addrs; i++) {
+                for (size_t j = 0; j < ps->n_ipv4_addrs; j++) {
                     /* When the netmask is applied, if the host portion is
                      * non-zero, the host can only use the specified
                      * address in the arp.spa.  If zero, the host is allowed
                      * to use any address in the subnet. */
-                    if (ps->ipv4_addrs[i].plen == 32
-                        || ps->ipv4_addrs[i].addr & ~ps->ipv4_addrs[i].mask) {
-                        ds_put_cstr(&match, ps->ipv4_addrs[i].addr_s);
+                    if (ps->ipv4_addrs[j].plen == 32
+                        || ps->ipv4_addrs[j].addr & ~ps->ipv4_addrs[j].mask) {
+                        ds_put_cstr(&match, ps->ipv4_addrs[j].addr_s);
                     } else {
                         ds_put_format(&match, "%s/%d",
-                                      ps->ipv4_addrs[i].network_s,
-                                      ps->ipv4_addrs[i].plen);
+                                      ps->ipv4_addrs[j].network_s,
+                                      ps->ipv4_addrs[j].plen);
                     }
                     ds_put_cstr(&match, ", ");
                 }
@@ -1265,26 +1267,26 @@ build_port_security_ip(enum ovn_pipeline pipeline, struct ovn_port *op,
                               op->json_key, ps->ea_s);
             }
 
-            for (int i = 0; i < ps->n_ipv4_addrs; i++) {
-                ovs_be32 mask = ps->ipv4_addrs[i].mask;
+            for (int j = 0; j < ps->n_ipv4_addrs; j++) {
+                ovs_be32 mask = ps->ipv4_addrs[j].mask;
                 /* When the netmask is applied, if the host portion is
                  * non-zero, the host can only use the specified
                  * address.  If zero, the host is allowed to use any
                  * address in the subnet.
                  */
-                if (ps->ipv4_addrs[i].plen == 32
-                    || ps->ipv4_addrs[i].addr & ~mask) {
-                    ds_put_format(&match, "%s", ps->ipv4_addrs[i].addr_s);
-                    if (pipeline == P_OUT && ps->ipv4_addrs[i].plen != 32) {
+                if (ps->ipv4_addrs[j].plen == 32
+                    || ps->ipv4_addrs[j].addr & ~mask) {
+                    ds_put_format(&match, "%s", ps->ipv4_addrs[j].addr_s);
+                    if (pipeline == P_OUT && ps->ipv4_addrs[j].plen != 32) {
                         /* Host is also allowed to receive packets to the
                          * broadcast address in the specified subnet. */
                         ds_put_format(&match, ", %s",
-                                      ps->ipv4_addrs[i].bcast_s);
+                                      ps->ipv4_addrs[j].bcast_s);
                     }
                 } else {
                     /* host portion is zero */
-                    ds_put_format(&match, "%s/%d", ps->ipv4_addrs[i].network_s,
-                                  ps->ipv4_addrs[i].plen);
+                    ds_put_format(&match, "%s/%d", ps->ipv4_addrs[j].network_s,
+                                  ps->ipv4_addrs[j].plen);
                 }
                 ds_put_cstr(&match, ", ");
             }
@@ -1559,48 +1561,77 @@ build_acls(struct ovn_datapath *od, struct hmap *lflows)
          * commit IP flows.  This is because, while the initiater's
          * direction may not have any stateful rules, the server's may
          * and then its return traffic would not have an associated
-         * conntrack entry and would return "+invalid". */
-        ovn_lflow_add(lflows, od, S_SWITCH_IN_ACL, 1, "ip",
-                      REGBIT_CONNTRACK_COMMIT" = 1; next;");
-        ovn_lflow_add(lflows, od, S_SWITCH_OUT_ACL, 1, "ip",
-                      REGBIT_CONNTRACK_COMMIT" = 1; next;");
+         * conntrack entry and would return "+invalid".
+         *
+         * We use "ct_commit" for a connection that is not already known
+         * by the connection tracker.  Once a connection is committed,
+         * subsequent packets will hit the flow at priority 0 that just
+         * uses "next;"
+         *
+         * We also check for established connections that have ct_label[0]
+         * set on them.  That's a connection that was disallowed, but is
+         * now allowed by policy again since it hit this default-allow flow.
+         * We need to set ct_label[0]=0 to let the connection continue,
+         * which will be done by ct_commit() in the "stateful" stage.
+         * Subsequent packets will hit the flow at priority 0 that just
+         * uses "next;". */
+        ovn_lflow_add(lflows, od, S_SWITCH_IN_ACL, 1,
+                      "ip && (!ct.est || (ct.est && ct_label[0] == 1))",
+                       REGBIT_CONNTRACK_COMMIT" = 1; next;");
+        ovn_lflow_add(lflows, od, S_SWITCH_OUT_ACL, 1,
+                      "ip && (!ct.est || (ct.est && ct_label[0] == 1))",
+                       REGBIT_CONNTRACK_COMMIT" = 1; next;");
 
         /* Ingress and Egress ACL Table (Priority 65535).
          *
-         * Always drop traffic that's in an invalid state.  This is
-         * enforced at a higher priority than ACLs can be defined. */
+         * Always drop traffic that's in an invalid state.  Also drop
+         * reply direction packets for connections that have been marked
+         * for deletion (bit 0 of ct_label is set).
+         *
+         * This is enforced at a higher priority than ACLs can be defined. */
         ovn_lflow_add(lflows, od, S_SWITCH_IN_ACL, UINT16_MAX,
-                      "ct.inv", "drop;");
+                      "ct.inv || (ct.est && ct.rpl && ct_label[0] == 1)",
+                      "drop;");
         ovn_lflow_add(lflows, od, S_SWITCH_OUT_ACL, UINT16_MAX,
-                      "ct.inv", "drop;");
+                      "ct.inv || (ct.est && ct.rpl && ct_label[0] == 1)",
+                      "drop;");
 
         /* Ingress and Egress ACL Table (Priority 65535).
          *
-         * Always allow traffic that is established to a committed
-         * conntrack entry.  This is enforced at a higher priority than
-         * ACLs can be defined. */
+         * Allow reply traffic that is part of an established
+         * conntrack entry that has not been marked for deletion
+         * (bit 0 of ct_label).  We only match traffic in the
+         * reply direction because we want traffic in the request
+         * direction to hit the currently defined policy from ACLs.
+         *
+         * This is enforced at a higher priority than ACLs can be defined. */
         ovn_lflow_add(lflows, od, S_SWITCH_IN_ACL, UINT16_MAX,
-                      "ct.est && !ct.rel && !ct.new && !ct.inv",
+                      "ct.est && !ct.rel && !ct.new && !ct.inv "
+                      "&& ct.rpl && ct_label[0] == 0",
                       "next;");
         ovn_lflow_add(lflows, od, S_SWITCH_OUT_ACL, UINT16_MAX,
-                      "ct.est && !ct.rel && !ct.new && !ct.inv",
+                      "ct.est && !ct.rel && !ct.new && !ct.inv "
+                      "&& ct.rpl && ct_label[0] == 0",
                       "next;");
 
         /* Ingress and Egress ACL Table (Priority 65535).
          *
-         * Always allow traffic that is related to an existing conntrack
-         * entry.  This is enforced at a higher priority than ACLs can
-         * be defined.
+         * Allow traffic that is related to an existing conntrack entry that
+         * has not been marked for deletion (bit 0 of ct_label).
+         *
+         * This is enforced at a higher priority than ACLs can be defined.
          *
          * NOTE: This does not support related data sessions (eg,
          * a dynamically negotiated FTP data channel), but will allow
          * related traffic such as an ICMP Port Unreachable through
          * that's generated from a non-listening UDP port.  */
         ovn_lflow_add(lflows, od, S_SWITCH_IN_ACL, UINT16_MAX,
-                      "!ct.est && ct.rel && !ct.new && !ct.inv",
+                      "!ct.est && ct.rel && !ct.new && !ct.inv "
+                      "&& ct_label[0] == 0",
                       "next;");
         ovn_lflow_add(lflows, od, S_SWITCH_OUT_ACL, UINT16_MAX,
-                      "!ct.est && ct.rel && !ct.new && !ct.inv",
+                      "!ct.est && ct.rel && !ct.new && !ct.inv "
+                      "&& ct_label[0] == 0",
                       "next;");
 
         /* Ingress and Egress ACL Table (Priority 65535).
@@ -1616,41 +1647,108 @@ build_acls(struct ovn_datapath *od, struct hmap *lflows)
         bool ingress = !strcmp(acl->direction, "from-lport") ? true :false;
         enum ovn_stage stage = ingress ? S_SWITCH_IN_ACL : S_SWITCH_OUT_ACL;
 
-        if (!strcmp(acl->action, "allow")) {
+        if (!strcmp(acl->action, "allow")
+            || !strcmp(acl->action, "allow-related")) {
             /* If there are any stateful flows, we must even commit "allow"
              * actions.  This is because, while the initiater's
              * direction may not have any stateful rules, the server's
              * may and then its return traffic would not have an
              * associated conntrack entry and would return "+invalid". */
-            const char *actions = has_stateful
-                                    ? REGBIT_CONNTRACK_COMMIT" = 1; next;"
-                                    : "next;";
-            ovn_lflow_add(lflows, od, stage,
-                          acl->priority + OVN_ACL_PRI_OFFSET,
-                          acl->match, actions);
-        } else if (!strcmp(acl->action, "allow-related")) {
+            if (!has_stateful) {
+                ovn_lflow_add(lflows, od, stage,
+                              acl->priority + OVN_ACL_PRI_OFFSET,
+                              acl->match, "next;");
+            } else {
+                struct ds match = DS_EMPTY_INITIALIZER;
+
+                /* Commit the connection tracking entry if it's a new
+                 * connection that matches this ACL.  After this commit,
+                 * the reply traffic is allowed by a flow we create at
+                 * priority 65535, defined earlier.
+                 *
+                 * It's also possible that a known connection was marked for
+                 * deletion after a policy was deleted, but the policy was
+                 * re-added while that connection is still known.  We catch
+                 * that case here and un-set ct_label[0] (which will be done
+                 * by ct_commit in the "stateful" stage) to indicate that the
+                 * connection should be allowed to resume.
+                 */
+                ds_put_format(&match, "((ct.new && !ct.est)"
+                                      " || (!ct.new && ct.est && !ct.rpl "
+                                           "&& ct_label[0] == 1)) "
+                                      "&& (%s)", acl->match);
+                ovn_lflow_add(lflows, od, stage,
+                              acl->priority + OVN_ACL_PRI_OFFSET,
+                              ds_cstr(&match),
+                               REGBIT_CONNTRACK_COMMIT" = 1; next;");
+
+                /* Match on traffic in the request direction for an established
+                 * connection tracking entry that has not been marked for
+                 * deletion.  There is no need to commit here, so we can just
+                 * proceed to the next table. We use this to ensure that this
+                 * connection is still allowed by the currently defined
+                 * policy. */
+                ds_clear(&match);
+                ds_put_format(&match,
+                              "!ct.new && ct.est && !ct.rpl"
+                              " && ct_label[0] == 0 && (%s)",
+                              acl->match);
+                ovn_lflow_add(lflows, od, stage,
+                              acl->priority + OVN_ACL_PRI_OFFSET,
+                              ds_cstr(&match), "next;");
+
+                ds_destroy(&match);
+            }
+        } else if (!strcmp(acl->action, "drop")
+                   || !strcmp(acl->action, "reject")) {
             struct ds match = DS_EMPTY_INITIALIZER;
 
-            /* Commit the connection tracking entry, which allows all
-             * other traffic related to this entry to flow due to the
-             * 65535 priority flow defined earlier. */
-            ds_put_format(&match, "ct.new && (%s)", acl->match);
-            ovn_lflow_add(lflows, od, stage,
-                          acl->priority + OVN_ACL_PRI_OFFSET,
-                          ds_cstr(&match),
-                          REGBIT_CONNTRACK_COMMIT" = 1; next;");
+            /* XXX Need to support "reject", treat it as "drop;" for now. */
+            if (!strcmp(acl->action, "reject")) {
+                VLOG_INFO("reject is not a supported action");
+            }
 
-            ds_destroy(&match);
-        } else if (!strcmp(acl->action, "drop")) {
-            ovn_lflow_add(lflows, od, stage,
-                          acl->priority + OVN_ACL_PRI_OFFSET,
-                          acl->match, "drop;");
-        } else if (!strcmp(acl->action, "reject")) {
-            /* xxx Need to support "reject". */
-            VLOG_INFO("reject is not a supported action");
-            ovn_lflow_add(lflows, od, stage,
-                          acl->priority + OVN_ACL_PRI_OFFSET,
-                          acl->match, "drop;");
+            /* The implementation of "drop" differs if stateful ACLs are in
+             * use for this datapath.  In that case, the actions differ
+             * depending on whether the connection was previously committed
+             * to the connection tracker with ct_commit. */
+            if (has_stateful) {
+                /* If the packet is not part of an established connection, then
+                 * we can simply drop it. */
+                ds_put_format(&match,
+                              "(!ct.est || (ct.est && ct_label[0] == 1)) "
+                              "&& (%s)",
+                              acl->match);
+                ovn_lflow_add(lflows, od, stage, acl->priority +
+                        OVN_ACL_PRI_OFFSET, ds_cstr(&match), "drop;");
+
+                /* For an existing connection without ct_label set, we've
+                 * encountered a policy change. ACLs previously allowed
+                 * this connection and we committed the connection tracking
+                 * entry.  Current policy says that we should drop this
+                 * connection.  First, we set bit 0 of ct_label to indicate
+                 * that this connection is set for deletion.  By not
+                 * specifying "next;", we implicitly drop the packet after
+                 * updating conntrack state.  We would normally defer
+                 * ct_commit() to the "stateful" stage, but since we're
+                 * dropping the packet, we go ahead and do it here. */
+                ds_clear(&match);
+                ds_put_format(&match,
+                              "ct.est && ct_label[0] == 0 && (%s)",
+                              acl->match);
+                ovn_lflow_add(lflows, od, stage,
+                              acl->priority + OVN_ACL_PRI_OFFSET,
+                              ds_cstr(&match), "ct_commit(ct_label=1/1);");
+
+                ds_destroy(&match);
+            } else {
+                /* There are no stateful ACLs in use on this datapath,
+                 * so a "drop" ACL is simply the "drop" logical flow action
+                 * in all cases. */
+                ovn_lflow_add(lflows, od, stage,
+                              acl->priority + OVN_ACL_PRI_OFFSET,
+                              acl->match, "drop;");
+            }
         }
     }
 }
@@ -1685,11 +1783,13 @@ build_stateful(struct ovn_datapath *od, struct hmap *lflows)
     ovn_lflow_add(lflows, od, S_SWITCH_OUT_STATEFUL, 0, "1", "next;");
 
     /* If REGBIT_CONNTRACK_COMMIT is set as 1, then the packets should be
-     * committed to conntrack. */
+     * committed to conntrack. We always set ct_label[0] to 0 here as
+     * any packet that makes it this far is part of a connection we
+     * want to allow to continue. */
     ovn_lflow_add(lflows, od, S_SWITCH_IN_STATEFUL, 100,
-                  REGBIT_CONNTRACK_COMMIT" == 1", "ct_commit; next;");
+                  REGBIT_CONNTRACK_COMMIT" == 1", "ct_commit(ct_label=0/1); next;");
     ovn_lflow_add(lflows, od, S_SWITCH_OUT_STATEFUL, 100,
-                  REGBIT_CONNTRACK_COMMIT" == 1", "ct_commit; next;");
+                  REGBIT_CONNTRACK_COMMIT" == 1", "ct_commit(ct_label=0/1); next;");
 
     /* If REGBIT_CONNTRACK_NAT is set as 1, then packets should just be sent
      * through nat (without committing).
@@ -2048,9 +2148,40 @@ lrport_is_enabled(const struct nbrec_logical_router_port *lrport)
     return !lrport->enabled || *lrport->enabled;
 }
 
+/* Returns a string of the IP address of the router port 'op' that
+ * overlaps with 'ip_s".  If one is not found, returns NULL.
+ *
+ * The caller must not free the returned string. */
+static const char *
+find_lrp_member_ip(const struct ovn_port *op, const char *ip_s)
+{
+    ovs_be32 ip;
+
+    if (!ip_parse(ip_s, &ip)) {
+        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+        VLOG_WARN_RL(&rl, "bad ip address %s", ip_s);
+        return NULL;
+    }
+
+    for (int i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) {
+        const struct ipv4_netaddr *na = &op->lrp_networks.ipv4_addrs[i];
+
+        if (!((na->network ^ ip) & na->mask)) {
+            /* There should be only 1 interface that matches the
+             * next hop.  Otherwise, it's a configuration error,
+             * because subnets of router's interfaces should NOT
+             * overlap. */
+            return na->addr_s;
+        }
+    }
+
+    return NULL;
+}
+
 static void
 add_route(struct hmap *lflows, const struct ovn_port *op,
-          const char *network_s, int plen, const char *gateway)
+          const char *lrp_addr_s, const char *network_s, int plen,
+          const char *gateway)
 {
     char *match = xasprintf("ip4.dst == %s/%d", network_s, plen);
 
@@ -2061,13 +2192,15 @@ add_route(struct hmap *lflows, const struct ovn_port *op,
     } else {
         ds_put_cstr(&actions, "ip4.dst");
     }
-    ds_put_format(&actions,
-                  "; "
+    ds_put_format(&actions, "; "
                   "reg1 = %s; "
-                  "eth.src = "ETH_ADDR_FMT"; "
+                  "eth.src = %s; "
                   "outport = %s; "
+                  "inport = \"\"; /* Allow sending out inport. */ "
                   "next;",
-                  op->ip_s, ETH_ADDR_ARGS(op->mac), op->json_key);
+                  lrp_addr_s,
+                  op->lrp_networks.ea_s,
+                  op->json_key);
 
     /* The priority here is calculated to implement longest-prefix-match
      * routing. */
@@ -2082,10 +2215,11 @@ build_static_route_flow(struct hmap *lflows, struct ovn_datapath *od,
                         struct hmap *ports,
                         const struct nbrec_logical_router_static_route *route)
 {
-    ovs_be32 prefix, next_hop, mask;
+    ovs_be32 prefix, nexthop, mask;
+    const char *lrp_addr_s;
 
     /* Verify that next hop is an IP address with 32 bits mask. */
-    char *error = ip_parse_masked(route->nexthop, &next_hop, &mask);
+    char *error = ip_parse_masked(route->nexthop, &nexthop, &mask);
     if (error || mask != OVS_BE32_MAX) {
         static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
         VLOG_WARN_RL(&rl, "bad next hop ip address %s", route->nexthop);
@@ -2113,6 +2247,7 @@ build_static_route_flow(struct hmap *lflows, struct ovn_datapath *od,
                          route->output_port, route->ip_prefix);
             return;
         }
+        lrp_addr_s = find_lrp_member_ip(out_port, route->nexthop);
     } else {
         /* output_port is not specified, find the
          * router port matching the next hop. */
@@ -2125,29 +2260,47 @@ build_static_route_flow(struct hmap *lflows, struct ovn_datapath *od,
                 continue;
             }
 
-            if (out_port->network
-                && !((out_port->network ^ next_hop) & out_port->mask)) {
-                /* There should be only 1 interface that matches the next hop.
-                 * Otherwise, it's a configuration error, because subnets of
-                 * router's interfaces should NOT overlap. */
+            lrp_addr_s = find_lrp_member_ip(out_port, route->nexthop);
+            if (lrp_addr_s) {
                 break;
             }
         }
-        if (i == od->nbr->n_ports) {
-            /* There is no matched out port. */
-            static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
-            VLOG_WARN_RL(&rl, "No path for static route %s; next hop %s",
-                         route->ip_prefix, route->nexthop);
-            return;
-        }
     }
 
-    char *prefix_s = xasprintf(IP_FMT, IP_ARGS(prefix));
-    add_route(lflows, out_port, prefix_s, ip_count_cidr_bits(mask),
-              route->nexthop);
+     if (!lrp_addr_s) {
+        /* There is no matched out port. */
+        static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1);
+        VLOG_WARN_RL(&rl, "No path for static route %s; next hop %s",
+                     route->ip_prefix, route->nexthop);
+        return;
+    }
+
+    char *prefix_s = xasprintf(IP_FMT, IP_ARGS(prefix & mask));
+    add_route(lflows, out_port, lrp_addr_s, prefix_s,
+              ip_count_cidr_bits(mask), route->nexthop);
     free(prefix_s);
 }
 
+static void
+op_put_networks(struct ds *ds, const struct ovn_port *op, bool add_bcast)
+{
+    if (!add_bcast && op->lrp_networks.n_ipv4_addrs == 1) {
+        ds_put_format(ds, "%s", op->lrp_networks.ipv4_addrs[0].addr_s);
+        return;
+    }
+
+    ds_put_cstr(ds, "{");
+    for (int i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) {
+        ds_put_format(ds, "%s, ", op->lrp_networks.ipv4_addrs[i].addr_s);
+        if (add_bcast) {
+            ds_put_format(ds, "%s, ", op->lrp_networks.ipv4_addrs[i].bcast_s);
+        }
+    }
+    ds_chomp(ds, ' ');
+    ds_chomp(ds, ',');
+    ds_put_cstr(ds, "}");
+}
+
 static void
 build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
                     struct hmap *lflows)
@@ -2185,9 +2338,8 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
         }
 
         ds_clear(&match);
-        ds_put_format(&match,
-            "(eth.mcast || eth.dst == "ETH_ADDR_FMT") && inport == %s",
-            ETH_ADDR_ARGS(op->mac), op->json_key);
+        ds_put_format(&match, "(eth.mcast || eth.dst == %s) && inport == %s",
+                      op->lrp_networks.ea_s, op->json_key);
         ovn_lflow_add(lflows, op->od, S_ROUTER_IN_ADMISSION, 50,
                       ds_cstr(&match), "next;");
     }
@@ -2220,10 +2372,6 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
         ovn_lflow_add(lflows, od, S_ROUTER_IN_IP_INPUT, 50,
                       "eth.bcast", "drop;");
 
-        /* Drop IP multicast. */
-        ovn_lflow_add(lflows, od, S_ROUTER_IN_IP_INPUT, 50,
-                      "ip4.mcast", "drop;");
-
         /* TTL discard.
          *
          * XXX Need to send ICMP time exceeded if !ip.later_frag. */
@@ -2246,7 +2394,8 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
          * owned by the router or a broadcast address known to the router
          * (priority 100). */
         ds_clear(&match);
-        ds_put_format(&match, "ip4.src == {%s, %s}", op->ip_s, op->bcast_s);
+        ds_put_cstr(&match, "ip4.src == ");
+        op_put_networks(&match, op, true);
         ovn_lflow_add(lflows, op->od, S_ROUTER_IN_IP_INPUT, 100,
                       ds_cstr(&match), "drop;");
 
@@ -2256,44 +2405,47 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
          * (i.e. the incoming locally attached net) does not matter.
          * The ip.ttl also does not matter (RFC1812 section 4.2.2.9) */
         ds_clear(&match);
-        ds_put_format(&match,
-                      "ip4.dst == %s && icmp4.type == 8 && icmp4.code == 0",
-                      op->ip_s);
+        ds_put_cstr(&match, "ip4.dst == ");
+        op_put_networks(&match, op, false);
+        ds_put_cstr(&match, " && icmp4.type == 8 && icmp4.code == 0");
+
         ds_clear(&actions);
         ds_put_format(&actions,
-            "ip4.dst = ip4.src; "
-            "ip4.src = %s; "
+            "ip4.dst <-> ip4.src; "
             "ip.ttl = 255; "
             "icmp4.type = 0; "
             "inport = \"\"; /* Allow sending out inport. */ "
-            "next; ",
-            op->ip_s);
+            "next; ");
         ovn_lflow_add(lflows, op->od, S_ROUTER_IN_IP_INPUT, 90,
                       ds_cstr(&match), ds_cstr(&actions));
 
         /* ARP reply.  These flows reply to ARP requests for the router's own
          * IP address. */
-        ds_clear(&match);
-        ds_put_format(&match, "inport == %s && arp.tpa == %s && arp.op == 1",
-                      op->json_key, op->ip_s);
-        ds_clear(&actions);
-        ds_put_format(&actions,
-            "eth.dst = eth.src; "
-            "eth.src = "ETH_ADDR_FMT"; "
-            "arp.op = 2; /* ARP reply */ "
-            "arp.tha = arp.sha; "
-            "arp.sha = "ETH_ADDR_FMT"; "
-            "arp.tpa = arp.spa; "
-            "arp.spa = %s; "
-            "outport = %s; "
-            "inport = \"\"; /* Allow sending out inport. */ "
-            "output;",
-            ETH_ADDR_ARGS(op->mac),
-            ETH_ADDR_ARGS(op->mac),
-            op->ip_s,
-            op->json_key);
-        ovn_lflow_add(lflows, op->od, S_ROUTER_IN_IP_INPUT, 90,
-                      ds_cstr(&match), ds_cstr(&actions));
+        for (int i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) {
+            ds_clear(&match);
+            ds_put_format(&match,
+                          "inport == %s && arp.tpa == %s && arp.op == 1",
+                          op->json_key, op->lrp_networks.ipv4_addrs[i].addr_s);
+
+            ds_clear(&actions);
+            ds_put_format(&actions,
+                "eth.dst = eth.src; "
+                "eth.src = %s; "
+                "arp.op = 2; /* ARP reply */ "
+                "arp.tha = arp.sha; "
+                "arp.sha = %s; "
+                "arp.tpa = arp.spa; "
+                "arp.spa = %s; "
+                "outport = %s; "
+                "inport = \"\"; /* Allow sending out inport. */ "
+                "output;",
+                op->lrp_networks.ea_s,
+                op->lrp_networks.ea_s,
+                op->lrp_networks.ipv4_addrs[i].addr_s,
+                op->json_key);
+            ovn_lflow_add(lflows, op->od, S_ROUTER_IN_IP_INPUT, 90,
+                          ds_cstr(&match), ds_cstr(&actions));
+        }
 
         /* ARP handling for external IP addresses.
          *
@@ -2320,20 +2472,21 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
             ds_put_format(&match,
                           "inport == %s && arp.tpa == "IP_FMT" && arp.op == 1",
                           op->json_key, IP_ARGS(ip));
+
             ds_clear(&actions);
             ds_put_format(&actions,
                 "eth.dst = eth.src; "
-                "eth.src = "ETH_ADDR_FMT"; "
+                "eth.src = %s; "
                 "arp.op = 2; /* ARP reply */ "
                 "arp.tha = arp.sha; "
-                "arp.sha = "ETH_ADDR_FMT"; "
+                "arp.sha = %s; "
                 "arp.tpa = arp.spa; "
                 "arp.spa = "IP_FMT"; "
                 "outport = %s; "
                 "inport = \"\"; /* Allow sending out inport. */ "
                 "output;",
-                ETH_ADDR_ARGS(op->mac),
-                ETH_ADDR_ARGS(op->mac),
+                op->lrp_networks.ea_s,
+                op->lrp_networks.ea_s,
                 IP_ARGS(ip),
                 op->json_key);
             ovn_lflow_add(lflows, op->od, S_ROUTER_IN_IP_INPUT, 90,
@@ -2342,7 +2495,8 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
 
         /* Drop IP traffic to this router, unless the router ip is used as
          * SNAT ip. */
-        bool snat_ip_is_router_ip = false;
+        ovs_be32 *nat_ips = xmalloc(sizeof *nat_ips * op->od->nbr->n_nat);
+        size_t n_nat_ips = 0;
         for (int i = 0; i < op->od->nbr->n_nat; i++) {
             const struct nbrec_nat *nat;
             ovs_be32 ip;
@@ -2359,18 +2513,33 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
                 continue;
             }
 
-            if (ip == op->ip) {
-                snat_ip_is_router_ip = true;
-                break;
+            nat_ips[n_nat_ips++] = ip;
+        }
+
+        ds_clear(&match);
+        ds_put_cstr(&match, "ip4.dst == {");
+        bool has_drop_ips = false;
+        for (int i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) {
+            for (int j = 0; j < n_nat_ips; j++) {
+                if (op->lrp_networks.ipv4_addrs[i].addr == nat_ips[j]) {
+                    continue;
+                }
             }
+            ds_put_format(&match, "%s, ",
+                          op->lrp_networks.ipv4_addrs[i].addr_s);
+            has_drop_ips = true;
         }
+        ds_chomp(&match, ' ');
+        ds_chomp(&match, ',');
+        ds_put_cstr(&match, "}");
 
-        if (!snat_ip_is_router_ip) {
-            ds_clear(&match);
-            ds_put_format(&match, "ip4.dst == %s", op->ip_s);
+        if (has_drop_ips) {
+            /* Drop IP traffic to this router. */
             ovn_lflow_add(lflows, op->od, S_ROUTER_IN_IP_INPUT, 60,
                           ds_cstr(&match), "drop;");
         }
+
+        free(nat_ips);
     }
 
     /* NAT in Gateway routers. */
@@ -2512,8 +2681,13 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
             continue;
         }
 
-        add_route(lflows, op, op->network_s, op->plen, NULL);
+        for (int i = 0; i < op->lrp_networks.n_ipv4_addrs; i++) {
+            add_route(lflows, op, op->lrp_networks.ipv4_addrs[i].addr_s,
+                      op->lrp_networks.ipv4_addrs[i].network_s,
+                      op->lrp_networks.ipv4_addrs[i].plen, NULL);
+        }
     }
+
     HMAP_FOR_EACH (od, key_node, datapaths) {
         if (!od->nbr) {
             continue;
@@ -2550,15 +2724,14 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
                     continue;
                 }
 
-                if (!peer->ip || !op->ip) {
-                    continue;
-                }
                 ds_clear(&match);
-                ds_put_format(&match, "outport == %s && reg0 == %s",
-                              peer->json_key, op->ip_s);
+                ds_put_format(&match, "outport == %s && reg0 == ",
+                              peer->json_key);
+                op_put_networks(&match, op, false);
+
                 ds_clear(&actions);
-                ds_put_format(&actions, "eth.dst = "ETH_ADDR_FMT"; next;",
-                              ETH_ADDR_ARGS(op->mac));
+                ds_put_format(&actions, "eth.dst = %s; next;",
+                              op->lrp_networks.ea_s);
                 ovn_lflow_add(lflows, peer->od, S_ROUTER_IN_ARP_RESOLVE,
                               100, ds_cstr(&match), ds_cstr(&actions));
             }
@@ -2570,8 +2743,9 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
              * router port, add an ARP entry in that router's pipeline. */
 
             for (size_t i = 0; i < op->n_lsp_addrs; i++) {
+                const char *ea_s = op->lsp_addrs[i].ea_s;
                 for (size_t j = 0; j < op->lsp_addrs[i].n_ipv4_addrs; j++) {
-                    ovs_be32 ip = op->lsp_addrs[i].ipv4_addrs[j].addr;
+                    const char *ip_s = op->lsp_addrs[i].ipv4_addrs[j].addr_s;
                     for (size_t k = 0; k < op->od->n_router_ports; k++) {
                         /* Get the Logical_Router_Port that the
                          * Logical_Switch_Port is connected to, as
@@ -2588,22 +2762,19 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
                             continue;
                         }
 
-                        /* Make sure that 'ip' is in 'peer''s network. */
-                        if ((ip ^ peer->network) & peer->mask) {
+                        if (!find_lrp_member_ip(peer, ip_s)) {
                             continue;
                         }
 
                         ds_clear(&match);
                         ds_put_format(&match, "outport == %s && reg0 == %s",
-                                      peer->json_key,
-                                      op->lsp_addrs[i].ipv4_addrs[j].addr_s);
+                                      peer->json_key, ip_s);
+
                         ds_clear(&actions);
-                        ds_put_format(&actions, "eth.dst = %s; next;",
-                                      op->lsp_addrs[i].ea_s);
+                        ds_put_format(&actions, "eth.dst = %s; next;", ea_s);
                         ovn_lflow_add(lflows, peer->od,
                                       S_ROUTER_IN_ARP_RESOLVE, 100,
                                       ds_cstr(&match), ds_cstr(&actions));
-                        break;
                     }
                 }
             }
@@ -2622,17 +2793,17 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
             }
 
             struct ovn_port *peer = ovn_port_find(ports, peer_name);
-            if (!peer || !peer->nbr || !peer->ip) {
+            if (!peer || !peer->nbr) {
                 continue;
             }
 
-            for (size_t j = 0; j < op->od->n_router_ports; j++) {
+            for (size_t i = 0; i < op->od->n_router_ports; i++) {
                 const char *router_port_name = smap_get(
-                                    &op->od->router_ports[j]->nbs->options,
+                                    &op->od->router_ports[i]->nbs->options,
                                     "router-port");
                 struct ovn_port *router_port = ovn_port_find(ports,
                                                              router_port_name);
-                if (!router_port || !router_port->nbr || !router_port->ip) {
+                if (!router_port || !router_port->nbr) {
                     continue;
                 }
 
@@ -2641,15 +2812,14 @@ build_lrouter_flows(struct hmap *datapaths, struct hmap *ports,
                    continue;
                 }
 
-                if (!router_port->ip) {
-                    continue;
-                }
                 ds_clear(&match);
-                ds_put_format(&match, "outport == %s && reg0 == %s",
-                              peer->json_key, router_port->ip_s);
+                ds_put_format(&match, "outport == %s && reg0 == ",
+                              peer->json_key);
+                op_put_networks(&match, router_port, false);
+
                 ds_clear(&actions);
-                ds_put_format(&actions, "eth.dst = "ETH_ADDR_FMT"; next;",
-                              ETH_ADDR_ARGS(router_port->mac));
+                ds_put_format(&actions, "eth.dst = %s; next;",
+                              router_port->lrp_networks.ea_s);
                 ovn_lflow_add(lflows, peer->od, S_ROUTER_IN_ARP_RESOLVE,
                               100, ds_cstr(&match), ds_cstr(&actions));
             }