#define MVNETA_CPU_MAP(cpu) (0x2540 + ((cpu) << 2))
#define MVNETA_CPU_RXQ_ACCESS_ALL_MASK 0x000000ff
#define MVNETA_CPU_TXQ_ACCESS_ALL_MASK 0x0000ff00
+#define MVNETA_CPU_RXQ_ACCESS(rxq) BIT(rxq)
+#define MVNETA_CPU_TXQ_ACCESS(txq) BIT(txq + 8)
#define MVNETA_RXQ_TIME_COAL_REG(q) (0x2580 + ((q) << 2))
-/* Exception Interrupt Port/Queue Cause register */
+/* Exception Interrupt Port/Queue Cause register
+ *
+ * Their behavior depend of the mapping done using the PCPX2Q
+ * registers. For a given CPU if the bit associated to a queue is not
+ * set, then for the register a read from this CPU will always return
+ * 0 and a write won't do anything
+ */
#define MVNETA_INTR_NEW_CAUSE 0x25a0
#define MVNETA_INTR_NEW_MASK 0x25a4
#define MVNETA_TX_MTU_MAX 0x3ffff
+/* The RSS lookup table actually has 256 entries but we do not use
+ * them yet
+ */
+#define MVNETA_RSS_LU_TABLE_SIZE 1
+
/* TSO header size */
#define TSO_HEADER_SIZE 128
struct mvneta_tx_queue *txqs;
struct net_device *dev;
struct notifier_block cpu_notifier;
+ int rxq_def;
/* Core clock */
struct clk *clk;
unsigned int use_inband_status:1;
u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)];
+
+ u32 indir[MVNETA_RSS_LU_TABLE_SIZE];
};
/* The mvneta_tx_desc and mvneta_rx_desc structures describe the
/* DMA address of TSO headers */
dma_addr_t tso_hdrs_phys;
+
+ /* Affinity mask for CPUs*/
+ cpumask_t affinity_mask;
};
struct mvneta_rx_queue {
mvreg_write(pp, MVNETA_TXQ_CMD, q_map);
/* Enable all initialized RXQs. */
- mvreg_write(pp, MVNETA_RXQ_CMD, BIT(rxq_def));
+ for (queue = 0; queue < rxq_number; queue++) {
+ struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
+
+ if (rxq->descs != NULL)
+ q_map |= (1 << queue);
+ }
+ mvreg_write(pp, MVNETA_RXQ_CMD, q_map);
}
/* Stop the Ethernet port activity */
int cpu;
int queue;
u32 val;
+ int max_cpu = num_present_cpus();
/* Clear all Cause registers */
mvreg_write(pp, MVNETA_INTR_NEW_CAUSE, 0);
/* Enable MBUS Retry bit16 */
mvreg_write(pp, MVNETA_MBUS_RETRY, 0x20);
- /* Set CPU queue access map - all CPUs have access to all RX
- * queues and to all TX queues
+ /* Set CPU queue access map. CPUs are assigned to the RX and
+ * TX queues modulo their number. If there is only one TX
+ * queue then it is assigned to the CPU associated to the
+ * default RX queue.
*/
- for_each_present_cpu(cpu)
- mvreg_write(pp, MVNETA_CPU_MAP(cpu),
- (MVNETA_CPU_RXQ_ACCESS_ALL_MASK |
- MVNETA_CPU_TXQ_ACCESS_ALL_MASK));
+ for_each_present_cpu(cpu) {
+ int rxq_map = 0, txq_map = 0;
+ int rxq, txq;
+
+ for (rxq = 0; rxq < rxq_number; rxq++)
+ if ((rxq % max_cpu) == cpu)
+ rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq);
+
+ for (txq = 0; txq < txq_number; txq++)
+ if ((txq % max_cpu) == cpu)
+ txq_map |= MVNETA_CPU_TXQ_ACCESS(txq);
+
+ /* With only one TX queue we configure a special case
+ * which will allow to get all the irq on a single
+ * CPU
+ */
+ if (txq_number == 1)
+ txq_map = (cpu == pp->rxq_def) ?
+ MVNETA_CPU_TXQ_ACCESS(1) : 0;
+
+ mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map);
+ }
/* Reset RX and TX DMAs */
mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET);
mvreg_write(pp, MVNETA_ACC_MODE, val);
/* Update val of portCfg register accordingly with all RxQueue types */
- val = MVNETA_PORT_CONFIG_DEFL_VALUE(rxq_def);
+ val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def);
mvreg_write(pp, MVNETA_PORT_CONFIG, val);
val = 0;
if (dev->flags & IFF_PROMISC) {
/* Accept all: Multicast + Unicast */
mvneta_rx_unicast_promisc_set(pp, 1);
- mvneta_set_ucast_table(pp, rxq_def);
- mvneta_set_special_mcast_table(pp, rxq_def);
- mvneta_set_other_mcast_table(pp, rxq_def);
+ mvneta_set_ucast_table(pp, pp->rxq_def);
+ mvneta_set_special_mcast_table(pp, pp->rxq_def);
+ mvneta_set_other_mcast_table(pp, pp->rxq_def);
} else {
/* Accept single Unicast */
mvneta_rx_unicast_promisc_set(pp, 0);
mvneta_set_ucast_table(pp, -1);
- mvneta_mac_addr_set(pp, dev->dev_addr, rxq_def);
+ mvneta_mac_addr_set(pp, dev->dev_addr, pp->rxq_def);
if (dev->flags & IFF_ALLMULTI) {
/* Accept all multicast */
- mvneta_set_special_mcast_table(pp, rxq_def);
- mvneta_set_other_mcast_table(pp, rxq_def);
+ mvneta_set_special_mcast_table(pp, pp->rxq_def);
+ mvneta_set_other_mcast_table(pp, pp->rxq_def);
} else {
/* Accept only initialized multicast */
mvneta_set_special_mcast_table(pp, -1);
if (!netdev_mc_empty(dev)) {
netdev_for_each_mc_addr(ha, dev) {
mvneta_mcast_addr_set(pp, ha->addr,
- rxq_def);
+ pp->rxq_def);
}
}
}
{
int rx_done = 0;
u32 cause_rx_tx;
+ int rx_queue;
struct mvneta_port *pp = netdev_priv(napi->dev);
struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
/* For the case where the last mvneta_poll did not process all
* RX packets
*/
+ rx_queue = fls(((cause_rx_tx >> 8) & 0xff));
+
cause_rx_tx |= port->cause_rx_tx;
- rx_done = mvneta_rx(pp, budget, &pp->rxqs[rxq_def]);
+
+ if (rx_queue) {
+ rx_queue = rx_queue - 1;
+ rx_done = mvneta_rx(pp, budget, &pp->rxqs[rx_queue]);
+ }
+
budget -= rx_done;
if (budget > 0) {
static int mvneta_txq_init(struct mvneta_port *pp,
struct mvneta_tx_queue *txq)
{
+ int cpu;
+
txq->size = pp->tx_ring_size;
/* A queue must always have room for at least one skb.
}
mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal);
+ /* Setup XPS mapping */
+ if (txq_number > 1)
+ cpu = txq->id % num_present_cpus();
+ else
+ cpu = pp->rxq_def % num_present_cpus();
+ cpumask_set_cpu(cpu, &txq->affinity_mask);
+ netif_set_xps_queue(pp->dev, &txq->affinity_mask, txq->id);
+
return 0;
}
/* Cleanup all Rx queues */
static void mvneta_cleanup_rxqs(struct mvneta_port *pp)
{
- mvneta_rxq_deinit(pp, &pp->rxqs[rxq_def]);
+ int queue;
+
+ for (queue = 0; queue < txq_number; queue++)
+ mvneta_rxq_deinit(pp, &pp->rxqs[queue]);
}
/* Init all Rx queues */
static int mvneta_setup_rxqs(struct mvneta_port *pp)
{
- int err = mvneta_rxq_init(pp, &pp->rxqs[rxq_def]);
- if (err) {
- netdev_err(pp->dev, "%s: can't create rxq=%d\n",
- __func__, rxq_def);
- mvneta_cleanup_rxqs(pp);
- return err;
+ int queue;
+
+ for (queue = 0; queue < rxq_number; queue++) {
+ int err = mvneta_rxq_init(pp, &pp->rxqs[queue]);
+
+ if (err) {
+ netdev_err(pp->dev, "%s: can't create rxq=%d\n",
+ __func__, queue);
+ mvneta_cleanup_rxqs(pp);
+ return err;
+ }
}
return 0;
return 0;
}
+static void mvneta_percpu_unmask_interrupt(void *arg)
+{
+ struct mvneta_port *pp = arg;
+
+ /* All the queue are unmasked, but actually only the ones
+ * maped to this CPU will be unmasked
+ */
+ mvreg_write(pp, MVNETA_INTR_NEW_MASK,
+ MVNETA_RX_INTR_MASK_ALL |
+ MVNETA_TX_INTR_MASK_ALL |
+ MVNETA_MISCINTR_INTR_MASK);
+}
+
+static void mvneta_percpu_mask_interrupt(void *arg)
+{
+ struct mvneta_port *pp = arg;
+
+ /* All the queue are masked, but actually only the ones
+ * maped to this CPU will be masked
+ */
+ mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0);
+ mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0);
+ mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0);
+}
+
static void mvneta_start_dev(struct mvneta_port *pp)
{
unsigned int cpu;
napi_enable(&port->napi);
}
- /* Unmask interrupts */
- mvreg_write(pp, MVNETA_INTR_NEW_MASK,
- MVNETA_RX_INTR_MASK(rxq_number) |
- MVNETA_TX_INTR_MASK(txq_number) |
- MVNETA_MISCINTR_INTR_MASK);
+ /* Unmask interrupts. It has to be done from each CPU */
+ for_each_online_cpu(cpu)
+ smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt,
+ pp, true);
mvreg_write(pp, MVNETA_INTR_MISC_MASK,
MVNETA_CAUSE_PHY_STATUS_CHANGE |
MVNETA_CAUSE_LINK_CHANGE |
mvneta_mac_addr_set(pp, dev->dev_addr, -1);
/* Set new addr in hw */
- mvneta_mac_addr_set(pp, sockaddr->sa_data, rxq_def);
+ mvneta_mac_addr_set(pp, sockaddr->sa_data, pp->rxq_def);
eth_commit_mac_addr_change(dev, addr);
return 0;
static void mvneta_percpu_elect(struct mvneta_port *pp)
{
- int online_cpu_idx, cpu, i = 0;
+ int online_cpu_idx, max_cpu, cpu, i = 0;
- online_cpu_idx = rxq_def % num_online_cpus();
+ online_cpu_idx = pp->rxq_def % num_online_cpus();
+ max_cpu = num_present_cpus();
for_each_online_cpu(cpu) {
+ int rxq_map = 0, txq_map = 0;
+ int rxq;
+
+ for (rxq = 0; rxq < rxq_number; rxq++)
+ if ((rxq % max_cpu) == cpu)
+ rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq);
+
if (i == online_cpu_idx)
- /* Enable per-CPU interrupt on the one CPU we
- * just elected
+ /* Map the default receive queue queue to the
+ * elected CPU
*/
- smp_call_function_single(cpu, mvneta_percpu_enable,
- pp, true);
+ rxq_map |= MVNETA_CPU_RXQ_ACCESS(pp->rxq_def);
+
+ /* We update the TX queue map only if we have one
+ * queue. In this case we associate the TX queue to
+ * the CPU bound to the default RX queue
+ */
+ if (txq_number == 1)
+ txq_map = (i == online_cpu_idx) ?
+ MVNETA_CPU_TXQ_ACCESS(1) : 0;
else
- /* Disable per-CPU interrupt on all the other CPU */
- smp_call_function_single(cpu, mvneta_percpu_disable,
- pp, true);
+ txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) &
+ MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
+
+ mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map);
+
+ /* Update the interrupt mask on each CPU according the
+ * new mapping
+ */
+ smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt,
+ pp, true);
i++;
+
}
};
mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0);
napi_enable(&port->napi);
+
+ /* Enable per-CPU interrupts on the CPU that is
+ * brought up.
+ */
+ smp_call_function_single(cpu, mvneta_percpu_enable,
+ pp, true);
+
/* Enable per-CPU interrupt on the one CPU we care
* about.
*/
mvneta_percpu_elect(pp);
- /* Unmask all ethernet port interrupts */
+ /* Unmask all ethernet port interrupts, as this
+ * notifier is called for each CPU then the CPU to
+ * Queue mapping is applied
+ */
mvreg_write(pp, MVNETA_INTR_NEW_MASK,
MVNETA_RX_INTR_MASK(rxq_number) |
MVNETA_TX_INTR_MASK(txq_number) |
static int mvneta_open(struct net_device *dev)
{
struct mvneta_port *pp = netdev_priv(dev);
- int ret;
+ int ret, cpu;
pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu);
pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) +
*/
mvneta_percpu_disable(pp);
- /* Elect a CPU to handle our RX queue interrupt */
- mvneta_percpu_elect(pp);
+ /* Enable per-CPU interrupt on all the CPU to handle our RX
+ * queue interrupts
+ */
+ for_each_online_cpu(cpu)
+ smp_call_function_single(cpu, mvneta_percpu_enable,
+ pp, true);
+
/* Register a CPU notifier to handle the case where our CPU
* might be taken offline.
return -EOPNOTSUPP;
}
+static u32 mvneta_ethtool_get_rxfh_indir_size(struct net_device *dev)
+{
+ return MVNETA_RSS_LU_TABLE_SIZE;
+}
+
+static int mvneta_ethtool_get_rxnfc(struct net_device *dev,
+ struct ethtool_rxnfc *info,
+ u32 *rules __always_unused)
+{
+ switch (info->cmd) {
+ case ETHTOOL_GRXRINGS:
+ info->data = rxq_number;
+ return 0;
+ case ETHTOOL_GRXFH:
+ return -EOPNOTSUPP;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int mvneta_config_rss(struct mvneta_port *pp)
+{
+ int cpu;
+ u32 val;
+
+ netif_tx_stop_all_queues(pp->dev);
+
+ for_each_online_cpu(cpu)
+ smp_call_function_single(cpu, mvneta_percpu_mask_interrupt,
+ pp, true);
+
+ /* We have to synchronise on the napi of each CPU */
+ for_each_online_cpu(cpu) {
+ struct mvneta_pcpu_port *pcpu_port =
+ per_cpu_ptr(pp->ports, cpu);
+
+ napi_synchronize(&pcpu_port->napi);
+ napi_disable(&pcpu_port->napi);
+ }
+
+ pp->rxq_def = pp->indir[0];
+
+ /* Update unicast mapping */
+ mvneta_set_rx_mode(pp->dev);
+
+ /* Update val of portCfg register accordingly with all RxQueue types */
+ val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def);
+ mvreg_write(pp, MVNETA_PORT_CONFIG, val);
+
+ /* Update the elected CPU matching the new rxq_def */
+ mvneta_percpu_elect(pp);
+
+ /* We have to synchronise on the napi of each CPU */
+ for_each_online_cpu(cpu) {
+ struct mvneta_pcpu_port *pcpu_port =
+ per_cpu_ptr(pp->ports, cpu);
+
+ napi_enable(&pcpu_port->napi);
+ }
+
+ netif_tx_start_all_queues(pp->dev);
+
+ return 0;
+}
+
+static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir,
+ const u8 *key, const u8 hfunc)
+{
+ struct mvneta_port *pp = netdev_priv(dev);
+ /* We require at least one supported parameter to be changed
+ * and no change in any of the unsupported parameters
+ */
+ if (key ||
+ (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
+ return -EOPNOTSUPP;
+
+ if (!indir)
+ return 0;
+
+ memcpy(pp->indir, indir, MVNETA_RSS_LU_TABLE_SIZE);
+
+ return mvneta_config_rss(pp);
+}
+
+static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
+ u8 *hfunc)
+{
+ struct mvneta_port *pp = netdev_priv(dev);
+
+ if (hfunc)
+ *hfunc = ETH_RSS_HASH_TOP;
+
+ if (!indir)
+ return 0;
+
+ memcpy(indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE);
+
+ return 0;
+}
+
static const struct net_device_ops mvneta_netdev_ops = {
.ndo_open = mvneta_open,
.ndo_stop = mvneta_stop,
.get_strings = mvneta_ethtool_get_strings,
.get_ethtool_stats = mvneta_ethtool_get_stats,
.get_sset_count = mvneta_ethtool_get_sset_count,
+ .get_rxfh_indir_size = mvneta_ethtool_get_rxfh_indir_size,
+ .get_rxnfc = mvneta_ethtool_get_rxnfc,
+ .get_rxfh = mvneta_ethtool_get_rxfh,
+ .set_rxfh = mvneta_ethtool_set_rxfh,
};
/* Initialize hw */
strcmp(managed, "in-band-status") == 0);
pp->cpu_notifier.notifier_call = mvneta_percpu_notifier;
+ pp->rxq_def = rxq_def;
+
+ pp->indir[0] = rxq_def;
+
pp->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(pp->clk)) {
err = PTR_ERR(pp->clk);
projects / cascardo / linux.git / blobdiff