drivers/infiniband/hw/hfi1/chip.c

   1 /*
   2  * Copyright(c) 2015, 2016 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 /*
  49  * This file contains all of the code that is specific to the HFI chip
  50  */
  51
  52 #include <linux/pci.h>
  53 #include <linux/delay.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/module.h>
  56
  57 #include "hfi.h"
  58 #include "trace.h"
  59 #include "mad.h"
  60 #include "pio.h"
  61 #include "sdma.h"
  62 #include "eprom.h"
  63 #include "efivar.h"
  64 #include "platform.h"
  65 #include "aspm.h"
  66
  67 #define NUM_IB_PORTS 1
  68
  69 uint kdeth_qp;
  70 module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
  71 MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
  72
  73 uint num_vls = HFI1_MAX_VLS_SUPPORTED;
  74 module_param(num_vls, uint, S_IRUGO);
  75 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
  76
  77 /*
  78  * Default time to aggregate two 10K packets from the idle state
  79  * (timer not running). The timer starts at the end of the first packet,
  80  * so only the time for one 10K packet and header plus a bit extra is needed.
  81  * 10 * 1024 + 64 header byte = 10304 byte
  82  * 10304 byte / 12.5 GB/s = 824.32ns
  83  */
  84 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
  85 module_param(rcv_intr_timeout, uint, S_IRUGO);
  86 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
  87
  88 uint rcv_intr_count = 16; /* same as qib */
  89 module_param(rcv_intr_count, uint, S_IRUGO);
  90 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
  91
  92 ushort link_crc_mask = SUPPORTED_CRCS;
  93 module_param(link_crc_mask, ushort, S_IRUGO);
  94 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
  95
  96 uint loopback;
  97 module_param_named(loopback, loopback, uint, S_IRUGO);
  98 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
  99
 100 /* Other driver tunables */
 101 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
 102 static ushort crc_14b_sideband = 1;
 103 static uint use_flr = 1;
 104 uint quick_linkup; /* skip LNI */
 105
 106 struct flag_table {
 107         u64 flag;       /* the flag */
 108         char *str;      /* description string */
 109         u16 extra;      /* extra information */
 110         u16 unused0;
 111         u32 unused1;
 112 };
 113
 114 /* str must be a string constant */
 115 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
 116 #define FLAG_ENTRY0(str, flag) {flag, str, 0}
 117
 118 /* Send Error Consequences */
 119 #define SEC_WRITE_DROPPED       0x1
 120 #define SEC_PACKET_DROPPED      0x2
 121 #define SEC_SC_HALTED           0x4     /* per-context only */
 122 #define SEC_SPC_FREEZE          0x8     /* per-HFI only */
 123
 124 #define MIN_KERNEL_KCTXTS         2
 125 #define FIRST_KERNEL_KCTXT        1
 126 /* sizes for both the QP and RSM map tables */
 127 #define NUM_MAP_ENTRIES         256
 128 #define NUM_MAP_REGS             32
 129
 130 /* Bit offset into the GUID which carries HFI id information */
 131 #define GUID_HFI_INDEX_SHIFT     39
 132
 133 /* extract the emulation revision */
 134 #define emulator_rev(dd) ((dd)->irev >> 8)
 135 /* parallel and serial emulation versions are 3 and 4 respectively */
 136 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
 137 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
 138
 139 /* RSM fields */
 140
 141 /* packet type */
 142 #define IB_PACKET_TYPE         2ull
 143 #define QW_SHIFT               6ull
 144 /* QPN[7..1] */
 145 #define QPN_WIDTH              7ull
 146
 147 /* LRH.BTH: QW 0, OFFSET 48 - for match */
 148 #define LRH_BTH_QW             0ull
 149 #define LRH_BTH_BIT_OFFSET     48ull
 150 #define LRH_BTH_OFFSET(off)    ((LRH_BTH_QW << QW_SHIFT) | (off))
 151 #define LRH_BTH_MATCH_OFFSET   LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
 152 #define LRH_BTH_SELECT
 153 #define LRH_BTH_MASK           3ull
 154 #define LRH_BTH_VALUE          2ull
 155
 156 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
 157 #define LRH_SC_QW              0ull
 158 #define LRH_SC_BIT_OFFSET      56ull
 159 #define LRH_SC_OFFSET(off)     ((LRH_SC_QW << QW_SHIFT) | (off))
 160 #define LRH_SC_MATCH_OFFSET    LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
 161 #define LRH_SC_MASK            128ull
 162 #define LRH_SC_VALUE           0ull
 163
 164 /* SC[n..0] QW 0, OFFSET 60 - for select */
 165 #define LRH_SC_SELECT_OFFSET  ((LRH_SC_QW << QW_SHIFT) | (60ull))
 166
 167 /* QPN[m+n:1] QW 1, OFFSET 1 */
 168 #define QPN_SELECT_OFFSET      ((1ull << QW_SHIFT) | (1ull))
 169
 170 /* defines to build power on SC2VL table */
 171 #define SC2VL_VAL( \
 172         num, \
 173         sc0, sc0val, \
 174         sc1, sc1val, \
 175         sc2, sc2val, \
 176         sc3, sc3val, \
 177         sc4, sc4val, \
 178         sc5, sc5val, \
 179         sc6, sc6val, \
 180         sc7, sc7val) \
 181 ( \
 182         ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
 183         ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
 184         ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
 185         ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
 186         ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
 187         ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
 188         ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
 189         ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT)   \
 190 )
 191
 192 #define DC_SC_VL_VAL( \
 193         range, \
 194         e0, e0val, \
 195         e1, e1val, \
 196         e2, e2val, \
 197         e3, e3val, \
 198         e4, e4val, \
 199         e5, e5val, \
 200         e6, e6val, \
 201         e7, e7val, \
 202         e8, e8val, \
 203         e9, e9val, \
 204         e10, e10val, \
 205         e11, e11val, \
 206         e12, e12val, \
 207         e13, e13val, \
 208         e14, e14val, \
 209         e15, e15val) \
 210 ( \
 211         ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
 212         ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
 213         ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
 214         ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
 215         ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
 216         ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
 217         ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
 218         ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
 219         ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
 220         ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
 221         ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
 222         ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
 223         ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
 224         ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
 225         ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
 226         ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
 227 )
 228
 229 /* all CceStatus sub-block freeze bits */
 230 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
 231                         | CCE_STATUS_RXE_FROZE_SMASK \
 232                         | CCE_STATUS_TXE_FROZE_SMASK \
 233                         | CCE_STATUS_TXE_PIO_FROZE_SMASK)
 234 /* all CceStatus sub-block TXE pause bits */
 235 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
 236                         | CCE_STATUS_TXE_PAUSED_SMASK \
 237                         | CCE_STATUS_SDMA_PAUSED_SMASK)
 238 /* all CceStatus sub-block RXE pause bits */
 239 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
 240
 241 /*
 242  * CCE Error flags.
 243  */
 244 static struct flag_table cce_err_status_flags[] = {
 245 /* 0*/  FLAG_ENTRY0("CceCsrParityErr",
 246                 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
 247 /* 1*/  FLAG_ENTRY0("CceCsrReadBadAddrErr",
 248                 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
 249 /* 2*/  FLAG_ENTRY0("CceCsrWriteBadAddrErr",
 250                 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
 251 /* 3*/  FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
 252                 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
 253 /* 4*/  FLAG_ENTRY0("CceTrgtAccessErr",
 254                 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
 255 /* 5*/  FLAG_ENTRY0("CceRspdDataParityErr",
 256                 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
 257 /* 6*/  FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
 258                 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
 259 /* 7*/  FLAG_ENTRY0("CceCsrCfgBusParityErr",
 260                 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
 261 /* 8*/  FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
 262                 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
 263 /* 9*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
 264             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
 265 /*10*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
 266             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
 267 /*11*/  FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
 268             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
 269 /*12*/  FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
 270                 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
 271 /*13*/  FLAG_ENTRY0("PcicRetryMemCorErr",
 272                 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
 273 /*14*/  FLAG_ENTRY0("PcicRetryMemCorErr",
 274                 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
 275 /*15*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 276                 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
 277 /*16*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 278                 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
 279 /*17*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 280                 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
 281 /*18*/  FLAG_ENTRY0("PcicCplDatQCorErr",
 282                 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
 283 /*19*/  FLAG_ENTRY0("PcicNPostHQParityErr",
 284                 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
 285 /*20*/  FLAG_ENTRY0("PcicNPostDatQParityErr",
 286                 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
 287 /*21*/  FLAG_ENTRY0("PcicRetryMemUncErr",
 288                 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
 289 /*22*/  FLAG_ENTRY0("PcicRetrySotMemUncErr",
 290                 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
 291 /*23*/  FLAG_ENTRY0("PcicPostHdQUncErr",
 292                 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
 293 /*24*/  FLAG_ENTRY0("PcicPostDatQUncErr",
 294                 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
 295 /*25*/  FLAG_ENTRY0("PcicCplHdQUncErr",
 296                 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
 297 /*26*/  FLAG_ENTRY0("PcicCplDatQUncErr",
 298                 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
 299 /*27*/  FLAG_ENTRY0("PcicTransmitFrontParityErr",
 300                 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
 301 /*28*/  FLAG_ENTRY0("PcicTransmitBackParityErr",
 302                 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
 303 /*29*/  FLAG_ENTRY0("PcicReceiveParityErr",
 304                 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
 305 /*30*/  FLAG_ENTRY0("CceTrgtCplTimeoutErr",
 306                 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
 307 /*31*/  FLAG_ENTRY0("LATriggered",
 308                 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
 309 /*32*/  FLAG_ENTRY0("CceSegReadBadAddrErr",
 310                 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
 311 /*33*/  FLAG_ENTRY0("CceSegWriteBadAddrErr",
 312                 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
 313 /*34*/  FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
 314                 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
 315 /*35*/  FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
 316                 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
 317 /*36*/  FLAG_ENTRY0("CceMsixTableCorErr",
 318                 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
 319 /*37*/  FLAG_ENTRY0("CceMsixTableUncErr",
 320                 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
 321 /*38*/  FLAG_ENTRY0("CceIntMapCorErr",
 322                 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
 323 /*39*/  FLAG_ENTRY0("CceIntMapUncErr",
 324                 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
 325 /*40*/  FLAG_ENTRY0("CceMsixCsrParityErr",
 326                 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
 327 /*41-63 reserved*/
 328 };
 329
 330 /*
 331  * Misc Error flags
 332  */
 333 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
 334 static struct flag_table misc_err_status_flags[] = {
 335 /* 0*/  FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
 336 /* 1*/  FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
 337 /* 2*/  FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
 338 /* 3*/  FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
 339 /* 4*/  FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
 340 /* 5*/  FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
 341 /* 6*/  FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
 342 /* 7*/  FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
 343 /* 8*/  FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
 344 /* 9*/  FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
 345 /*10*/  FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
 346 /*11*/  FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
 347 /*12*/  FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
 348 };
 349
 350 /*
 351  * TXE PIO Error flags and consequences
 352  */
 353 static struct flag_table pio_err_status_flags[] = {
 354 /* 0*/  FLAG_ENTRY("PioWriteBadCtxt",
 355         SEC_WRITE_DROPPED,
 356         SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
 357 /* 1*/  FLAG_ENTRY("PioWriteAddrParity",
 358         SEC_SPC_FREEZE,
 359         SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
 360 /* 2*/  FLAG_ENTRY("PioCsrParity",
 361         SEC_SPC_FREEZE,
 362         SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
 363 /* 3*/  FLAG_ENTRY("PioSbMemFifo0",
 364         SEC_SPC_FREEZE,
 365         SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
 366 /* 4*/  FLAG_ENTRY("PioSbMemFifo1",
 367         SEC_SPC_FREEZE,
 368         SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
 369 /* 5*/  FLAG_ENTRY("PioPccFifoParity",
 370         SEC_SPC_FREEZE,
 371         SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
 372 /* 6*/  FLAG_ENTRY("PioPecFifoParity",
 373         SEC_SPC_FREEZE,
 374         SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
 375 /* 7*/  FLAG_ENTRY("PioSbrdctlCrrelParity",
 376         SEC_SPC_FREEZE,
 377         SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
 378 /* 8*/  FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
 379         SEC_SPC_FREEZE,
 380         SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
 381 /* 9*/  FLAG_ENTRY("PioPktEvictFifoParityErr",
 382         SEC_SPC_FREEZE,
 383         SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
 384 /*10*/  FLAG_ENTRY("PioSmPktResetParity",
 385         SEC_SPC_FREEZE,
 386         SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
 387 /*11*/  FLAG_ENTRY("PioVlLenMemBank0Unc",
 388         SEC_SPC_FREEZE,
 389         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
 390 /*12*/  FLAG_ENTRY("PioVlLenMemBank1Unc",
 391         SEC_SPC_FREEZE,
 392         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
 393 /*13*/  FLAG_ENTRY("PioVlLenMemBank0Cor",
 394         0,
 395         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
 396 /*14*/  FLAG_ENTRY("PioVlLenMemBank1Cor",
 397         0,
 398         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
 399 /*15*/  FLAG_ENTRY("PioCreditRetFifoParity",
 400         SEC_SPC_FREEZE,
 401         SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
 402 /*16*/  FLAG_ENTRY("PioPpmcPblFifo",
 403         SEC_SPC_FREEZE,
 404         SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
 405 /*17*/  FLAG_ENTRY("PioInitSmIn",
 406         0,
 407         SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
 408 /*18*/  FLAG_ENTRY("PioPktEvictSmOrArbSm",
 409         SEC_SPC_FREEZE,
 410         SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
 411 /*19*/  FLAG_ENTRY("PioHostAddrMemUnc",
 412         SEC_SPC_FREEZE,
 413         SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
 414 /*20*/  FLAG_ENTRY("PioHostAddrMemCor",
 415         0,
 416         SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
 417 /*21*/  FLAG_ENTRY("PioWriteDataParity",
 418         SEC_SPC_FREEZE,
 419         SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
 420 /*22*/  FLAG_ENTRY("PioStateMachine",
 421         SEC_SPC_FREEZE,
 422         SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
 423 /*23*/  FLAG_ENTRY("PioWriteQwValidParity",
 424         SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
 425         SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
 426 /*24*/  FLAG_ENTRY("PioBlockQwCountParity",
 427         SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
 428         SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
 429 /*25*/  FLAG_ENTRY("PioVlfVlLenParity",
 430         SEC_SPC_FREEZE,
 431         SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
 432 /*26*/  FLAG_ENTRY("PioVlfSopParity",
 433         SEC_SPC_FREEZE,
 434         SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
 435 /*27*/  FLAG_ENTRY("PioVlFifoParity",
 436         SEC_SPC_FREEZE,
 437         SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
 438 /*28*/  FLAG_ENTRY("PioPpmcBqcMemParity",
 439         SEC_SPC_FREEZE,
 440         SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
 441 /*29*/  FLAG_ENTRY("PioPpmcSopLen",
 442         SEC_SPC_FREEZE,
 443         SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
 444 /*30-31 reserved*/
 445 /*32*/  FLAG_ENTRY("PioCurrentFreeCntParity",
 446         SEC_SPC_FREEZE,
 447         SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
 448 /*33*/  FLAG_ENTRY("PioLastReturnedCntParity",
 449         SEC_SPC_FREEZE,
 450         SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
 451 /*34*/  FLAG_ENTRY("PioPccSopHeadParity",
 452         SEC_SPC_FREEZE,
 453         SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
 454 /*35*/  FLAG_ENTRY("PioPecSopHeadParityErr",
 455         SEC_SPC_FREEZE,
 456         SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
 457 /*36-63 reserved*/
 458 };
 459
 460 /* TXE PIO errors that cause an SPC freeze */
 461 #define ALL_PIO_FREEZE_ERR \
 462         (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
 463         | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
 464         | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
 465         | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
 466         | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
 467         | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
 468         | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
 469         | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
 470         | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
 471         | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
 472         | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
 473         | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
 474         | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
 475         | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
 476         | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
 477         | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
 478         | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
 479         | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
 480         | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
 481         | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
 482         | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
 483         | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
 484         | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
 485         | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
 486         | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
 487         | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
 488         | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
 489         | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
 490         | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
 491
 492 /*
 493  * TXE SDMA Error flags
 494  */
 495 static struct flag_table sdma_err_status_flags[] = {
 496 /* 0*/  FLAG_ENTRY0("SDmaRpyTagErr",
 497                 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
 498 /* 1*/  FLAG_ENTRY0("SDmaCsrParityErr",
 499                 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
 500 /* 2*/  FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
 501                 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
 502 /* 3*/  FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
 503                 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
 504 /*04-63 reserved*/
 505 };
 506
 507 /* TXE SDMA errors that cause an SPC freeze */
 508 #define ALL_SDMA_FREEZE_ERR  \
 509                 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
 510                 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
 511                 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
 512
 513 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
 514 #define PORT_DISCARD_EGRESS_ERRS \
 515         (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
 516         | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
 517         | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
 518
 519 /*
 520  * TXE Egress Error flags
 521  */
 522 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
 523 static struct flag_table egress_err_status_flags[] = {
 524 /* 0*/  FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
 525 /* 1*/  FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
 526 /* 2 reserved */
 527 /* 3*/  FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
 528                 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
 529 /* 4*/  FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
 530 /* 5*/  FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
 531 /* 6 reserved */
 532 /* 7*/  FLAG_ENTRY0("TxPioLaunchIntfParityErr",
 533                 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
 534 /* 8*/  FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
 535                 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
 536 /* 9-10 reserved */
 537 /*11*/  FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
 538                 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
 539 /*12*/  FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
 540 /*13*/  FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
 541 /*14*/  FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
 542 /*15*/  FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
 543 /*16*/  FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
 544                 SEES(TX_SDMA0_DISALLOWED_PACKET)),
 545 /*17*/  FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
 546                 SEES(TX_SDMA1_DISALLOWED_PACKET)),
 547 /*18*/  FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
 548                 SEES(TX_SDMA2_DISALLOWED_PACKET)),
 549 /*19*/  FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
 550                 SEES(TX_SDMA3_DISALLOWED_PACKET)),
 551 /*20*/  FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
 552                 SEES(TX_SDMA4_DISALLOWED_PACKET)),
 553 /*21*/  FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
 554                 SEES(TX_SDMA5_DISALLOWED_PACKET)),
 555 /*22*/  FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
 556                 SEES(TX_SDMA6_DISALLOWED_PACKET)),
 557 /*23*/  FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
 558                 SEES(TX_SDMA7_DISALLOWED_PACKET)),
 559 /*24*/  FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
 560                 SEES(TX_SDMA8_DISALLOWED_PACKET)),
 561 /*25*/  FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
 562                 SEES(TX_SDMA9_DISALLOWED_PACKET)),
 563 /*26*/  FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
 564                 SEES(TX_SDMA10_DISALLOWED_PACKET)),
 565 /*27*/  FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
 566                 SEES(TX_SDMA11_DISALLOWED_PACKET)),
 567 /*28*/  FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
 568                 SEES(TX_SDMA12_DISALLOWED_PACKET)),
 569 /*29*/  FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
 570                 SEES(TX_SDMA13_DISALLOWED_PACKET)),
 571 /*30*/  FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
 572                 SEES(TX_SDMA14_DISALLOWED_PACKET)),
 573 /*31*/  FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
 574                 SEES(TX_SDMA15_DISALLOWED_PACKET)),
 575 /*32*/  FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
 576                 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
 577 /*33*/  FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
 578                 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
 579 /*34*/  FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
 580                 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
 581 /*35*/  FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
 582                 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
 583 /*36*/  FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
 584                 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
 585 /*37*/  FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
 586                 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
 587 /*38*/  FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
 588                 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
 589 /*39*/  FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
 590                 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
 591 /*40*/  FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
 592                 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
 593 /*41*/  FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
 594 /*42*/  FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
 595 /*43*/  FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
 596 /*44*/  FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
 597 /*45*/  FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
 598 /*46*/  FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
 599 /*47*/  FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
 600 /*48*/  FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
 601 /*49*/  FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
 602 /*50*/  FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
 603 /*51*/  FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
 604 /*52*/  FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
 605 /*53*/  FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
 606 /*54*/  FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
 607 /*55*/  FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
 608 /*56*/  FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
 609 /*57*/  FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
 610 /*58*/  FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
 611 /*59*/  FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
 612 /*60*/  FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
 613 /*61*/  FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
 614 /*62*/  FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
 615                 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
 616 /*63*/  FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
 617                 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
 618 };
 619
 620 /*
 621  * TXE Egress Error Info flags
 622  */
 623 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
 624 static struct flag_table egress_err_info_flags[] = {
 625 /* 0*/  FLAG_ENTRY0("Reserved", 0ull),
 626 /* 1*/  FLAG_ENTRY0("VLErr", SEEI(VL)),
 627 /* 2*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
 628 /* 3*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
 629 /* 4*/  FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
 630 /* 5*/  FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
 631 /* 6*/  FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
 632 /* 7*/  FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
 633 /* 8*/  FLAG_ENTRY0("RawErr", SEEI(RAW)),
 634 /* 9*/  FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
 635 /*10*/  FLAG_ENTRY0("GRHErr", SEEI(GRH)),
 636 /*11*/  FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
 637 /*12*/  FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
 638 /*13*/  FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
 639 /*14*/  FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
 640 /*15*/  FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
 641 /*16*/  FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
 642 /*17*/  FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
 643 /*18*/  FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
 644 /*19*/  FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
 645 /*20*/  FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
 646 /*21*/  FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
 647 };
 648
 649 /* TXE Egress errors that cause an SPC freeze */
 650 #define ALL_TXE_EGRESS_FREEZE_ERR \
 651         (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
 652         | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
 653         | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
 654         | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
 655         | SEES(TX_LAUNCH_CSR_PARITY) \
 656         | SEES(TX_SBRD_CTL_CSR_PARITY) \
 657         | SEES(TX_CONFIG_PARITY) \
 658         | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
 659         | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
 660         | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
 661         | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
 662         | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
 663         | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
 664         | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
 665         | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
 666         | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
 667         | SEES(TX_CREDIT_RETURN_PARITY))
 668
 669 /*
 670  * TXE Send error flags
 671  */
 672 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
 673 static struct flag_table send_err_status_flags[] = {
 674 /* 0*/  FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
 675 /* 1*/  FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
 676 /* 2*/  FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
 677 };
 678
 679 /*
 680  * TXE Send Context Error flags and consequences
 681  */
 682 static struct flag_table sc_err_status_flags[] = {
 683 /* 0*/  FLAG_ENTRY("InconsistentSop",
 684                 SEC_PACKET_DROPPED | SEC_SC_HALTED,
 685                 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
 686 /* 1*/  FLAG_ENTRY("DisallowedPacket",
 687                 SEC_PACKET_DROPPED | SEC_SC_HALTED,
 688                 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
 689 /* 2*/  FLAG_ENTRY("WriteCrossesBoundary",
 690                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 691                 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
 692 /* 3*/  FLAG_ENTRY("WriteOverflow",
 693                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 694                 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
 695 /* 4*/  FLAG_ENTRY("WriteOutOfBounds",
 696                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 697                 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
 698 /* 5-63 reserved*/
 699 };
 700
 701 /*
 702  * RXE Receive Error flags
 703  */
 704 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
 705 static struct flag_table rxe_err_status_flags[] = {
 706 /* 0*/  FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
 707 /* 1*/  FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
 708 /* 2*/  FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
 709 /* 3*/  FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
 710 /* 4*/  FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
 711 /* 5*/  FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
 712 /* 6*/  FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
 713 /* 7*/  FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
 714 /* 8*/  FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
 715 /* 9*/  FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
 716 /*10*/  FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
 717 /*11*/  FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
 718 /*12*/  FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
 719 /*13*/  FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
 720 /*14*/  FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
 721 /*15*/  FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
 722 /*16*/  FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
 723                 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
 724 /*17*/  FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
 725 /*18*/  FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
 726 /*19*/  FLAG_ENTRY0("RxRbufBlockListReadUncErr",
 727                 RXES(RBUF_BLOCK_LIST_READ_UNC)),
 728 /*20*/  FLAG_ENTRY0("RxRbufBlockListReadCorErr",
 729                 RXES(RBUF_BLOCK_LIST_READ_COR)),
 730 /*21*/  FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
 731                 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
 732 /*22*/  FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
 733                 RXES(RBUF_CSR_QENT_CNT_PARITY)),
 734 /*23*/  FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
 735                 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
 736 /*24*/  FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
 737                 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
 738 /*25*/  FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
 739 /*26*/  FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
 740 /*27*/  FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
 741                 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
 742 /*28*/  FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
 743 /*29*/  FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
 744 /*30*/  FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
 745 /*31*/  FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
 746 /*32*/  FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
 747 /*33*/  FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
 748 /*34*/  FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
 749 /*35*/  FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
 750                 RXES(RBUF_FL_INITDONE_PARITY)),
 751 /*36*/  FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
 752                 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
 753 /*37*/  FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
 754 /*38*/  FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
 755 /*39*/  FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
 756 /*40*/  FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
 757                 RXES(LOOKUP_DES_PART1_UNC_COR)),
 758 /*41*/  FLAG_ENTRY0("RxLookupDesPart2ParityErr",
 759                 RXES(LOOKUP_DES_PART2_PARITY)),
 760 /*42*/  FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
 761 /*43*/  FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
 762 /*44*/  FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
 763 /*45*/  FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
 764 /*46*/  FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
 765 /*47*/  FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
 766 /*48*/  FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
 767 /*49*/  FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
 768 /*50*/  FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
 769 /*51*/  FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
 770 /*52*/  FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
 771 /*53*/  FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
 772 /*54*/  FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
 773 /*55*/  FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
 774 /*56*/  FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
 775 /*57*/  FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
 776 /*58*/  FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
 777 /*59*/  FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
 778 /*60*/  FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
 779 /*61*/  FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
 780 /*62*/  FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
 781 /*63*/  FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
 782 };
 783
 784 /* RXE errors that will trigger an SPC freeze */
 785 #define ALL_RXE_FREEZE_ERR  \
 786         (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
 787         | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
 788         | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
 789         | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
 790         | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
 791         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
 792         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
 793         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
 794         | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
 795         | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
 796         | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
 797         | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
 798         | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
 799         | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
 800         | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
 801         | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
 802         | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
 803         | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
 804         | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
 805         | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
 806         | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
 807         | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
 808         | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
 809         | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
 810         | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
 811         | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
 812         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
 813         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
 814         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
 815         | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
 816         | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
 817         | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
 818         | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
 819         | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
 820         | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
 821         | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
 822         | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
 823         | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
 824         | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
 825         | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
 826         | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
 827         | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
 828         | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
 829         | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
 830
 831 #define RXE_FREEZE_ABORT_MASK \
 832         (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
 833         RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
 834         RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
 835
 836 /*
 837  * DCC Error Flags
 838  */
 839 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK
 840 static struct flag_table dcc_err_flags[] = {
 841         FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
 842         FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
 843         FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
 844         FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
 845         FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
 846         FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
 847         FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
 848         FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
 849         FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
 850         FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
 851         FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
 852         FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
 853         FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
 854         FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
 855         FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
 856         FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
 857         FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
 858         FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
 859         FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
 860         FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
 861         FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
 862         FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
 863         FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
 864         FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
 865         FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
 866         FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
 867         FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
 868         FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
 869         FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
 870         FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
 871         FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
 872         FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
 873         FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
 874         FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
 875         FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
 876         FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
 877         FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
 878         FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
 879         FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
 880         FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
 881         FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
 882         FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
 883         FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
 884         FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
 885         FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
 886         FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
 887 };
 888
 889 /*
 890  * LCB error flags
 891  */
 892 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
 893 static struct flag_table lcb_err_flags[] = {
 894 /* 0*/  FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
 895 /* 1*/  FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
 896 /* 2*/  FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
 897 /* 3*/  FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
 898                 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
 899 /* 4*/  FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
 900 /* 5*/  FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
 901 /* 6*/  FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
 902 /* 7*/  FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
 903 /* 8*/  FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
 904 /* 9*/  FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
 905 /*10*/  FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
 906 /*11*/  FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
 907 /*12*/  FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
 908 /*13*/  FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
 909                 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
 910 /*14*/  FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
 911 /*15*/  FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
 912 /*16*/  FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
 913 /*17*/  FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
 914 /*18*/  FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
 915 /*19*/  FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
 916                 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
 917 /*20*/  FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
 918 /*21*/  FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
 919 /*22*/  FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
 920 /*23*/  FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
 921 /*24*/  FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
 922 /*25*/  FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
 923 /*26*/  FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
 924                 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
 925 /*27*/  FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
 926 /*28*/  FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
 927                 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
 928 /*29*/  FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
 929                 LCBE(REDUNDANT_FLIT_PARITY_ERR))
 930 };
 931
 932 /*
 933  * DC8051 Error Flags
 934  */
 935 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
 936 static struct flag_table dc8051_err_flags[] = {
 937         FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
 938         FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
 939         FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
 940         FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
 941         FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
 942         FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
 943         FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
 944         FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
 945         FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
 946                     D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
 947         FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
 948 };
 949
 950 /*
 951  * DC8051 Information Error flags
 952  *
 953  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
 954  */
 955 static struct flag_table dc8051_info_err_flags[] = {
 956         FLAG_ENTRY0("Spico ROM check failed",  SPICO_ROM_FAILED),
 957         FLAG_ENTRY0("Unknown frame received",  UNKNOWN_FRAME),
 958         FLAG_ENTRY0("Target BER not met",      TARGET_BER_NOT_MET),
 959         FLAG_ENTRY0("Serdes internal loopback failure",
 960                     FAILED_SERDES_INTERNAL_LOOPBACK),
 961         FLAG_ENTRY0("Failed SerDes init",      FAILED_SERDES_INIT),
 962         FLAG_ENTRY0("Failed LNI(Polling)",     FAILED_LNI_POLLING),
 963         FLAG_ENTRY0("Failed LNI(Debounce)",    FAILED_LNI_DEBOUNCE),
 964         FLAG_ENTRY0("Failed LNI(EstbComm)",    FAILED_LNI_ESTBCOMM),
 965         FLAG_ENTRY0("Failed LNI(OptEq)",       FAILED_LNI_OPTEQ),
 966         FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
 967         FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
 968         FLAG_ENTRY0("Failed LNI(ConfigLT)",    FAILED_LNI_CONFIGLT),
 969         FLAG_ENTRY0("Host Handshake Timeout",  HOST_HANDSHAKE_TIMEOUT)
 970 };
 971
 972 /*
 973  * DC8051 Information Host Information flags
 974  *
 975  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
 976  */
 977 static struct flag_table dc8051_info_host_msg_flags[] = {
 978         FLAG_ENTRY0("Host request done", 0x0001),
 979         FLAG_ENTRY0("BC SMA message", 0x0002),
 980         FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
 981         FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
 982         FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
 983         FLAG_ENTRY0("External device config request", 0x0020),
 984         FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
 985         FLAG_ENTRY0("LinkUp achieved", 0x0080),
 986         FLAG_ENTRY0("Link going down", 0x0100),
 987 };
 988
 989 static u32 encoded_size(u32 size);
 990 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
 991 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
 992 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
 993                                u8 *continuous);
 994 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
 995                                   u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
 996 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
 997                                       u8 *remote_tx_rate, u16 *link_widths);
 998 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
 999                                      u8 *flag_bits, u16 *link_widths);
1000 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1001                                   u8 *device_rev);
1002 static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed);
1003 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1004 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1005                             u8 *tx_polarity_inversion,
1006                             u8 *rx_polarity_inversion, u8 *max_rate);
1007 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1008                                 unsigned int context, u64 err_status);
1009 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1010 static void handle_dcc_err(struct hfi1_devdata *dd,
1011                            unsigned int context, u64 err_status);
1012 static void handle_lcb_err(struct hfi1_devdata *dd,
1013                            unsigned int context, u64 err_status);
1014 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1015 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1016 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1017 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1018 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1019 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1020 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1021 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1022 static void set_partition_keys(struct hfi1_pportdata *);
1023 static const char *link_state_name(u32 state);
1024 static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1025                                           u32 state);
1026 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1027                            u64 *out_data);
1028 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1029 static int thermal_init(struct hfi1_devdata *dd);
1030
1031 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1032                                   int msecs);
1033 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1034 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
1035 static void handle_temp_err(struct hfi1_devdata *);
1036 static void dc_shutdown(struct hfi1_devdata *);
1037 static void dc_start(struct hfi1_devdata *);
1038 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1039                            unsigned int *np);
1040 static void remove_full_mgmt_pkey(struct hfi1_pportdata *ppd);
1041
1042 /*
1043  * Error interrupt table entry.  This is used as input to the interrupt
1044  * "clear down" routine used for all second tier error interrupt register.
1045  * Second tier interrupt registers have a single bit representing them
1046  * in the top-level CceIntStatus.
1047  */
1048 struct err_reg_info {
1049         u32 status;             /* status CSR offset */
1050         u32 clear;              /* clear CSR offset */
1051         u32 mask;               /* mask CSR offset */
1052         void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1053         const char *desc;
1054 };
1055
1056 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START)
1057 #define NUM_DC_ERRS (IS_DC_END - IS_DC_START)
1058 #define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START)
1059
1060 /*
1061  * Helpers for building HFI and DC error interrupt table entries.  Different
1062  * helpers are needed because of inconsistent register names.
1063  */
1064 #define EE(reg, handler, desc) \
1065         { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1066                 handler, desc }
1067 #define DC_EE1(reg, handler, desc) \
1068         { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1069 #define DC_EE2(reg, handler, desc) \
1070         { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1071
1072 /*
1073  * Table of the "misc" grouping of error interrupts.  Each entry refers to
1074  * another register containing more information.
1075  */
1076 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1077 /* 0*/  EE(CCE_ERR,             handle_cce_err,    "CceErr"),
1078 /* 1*/  EE(RCV_ERR,             handle_rxe_err,    "RxeErr"),
1079 /* 2*/  EE(MISC_ERR,    handle_misc_err,   "MiscErr"),
1080 /* 3*/  { 0, 0, 0, NULL }, /* reserved */
1081 /* 4*/  EE(SEND_PIO_ERR,    handle_pio_err,    "PioErr"),
1082 /* 5*/  EE(SEND_DMA_ERR,    handle_sdma_err,   "SDmaErr"),
1083 /* 6*/  EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1084 /* 7*/  EE(SEND_ERR,    handle_txe_err,    "TxeErr")
1085         /* the rest are reserved */
1086 };
1087
1088 /*
1089  * Index into the Various section of the interrupt sources
1090  * corresponding to the Critical Temperature interrupt.
1091  */
1092 #define TCRIT_INT_SOURCE 4
1093
1094 /*
1095  * SDMA error interrupt entry - refers to another register containing more
1096  * information.
1097  */
1098 static const struct err_reg_info sdma_eng_err =
1099         EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1100
1101 static const struct err_reg_info various_err[NUM_VARIOUS] = {
1102 /* 0*/  { 0, 0, 0, NULL }, /* PbcInt */
1103 /* 1*/  { 0, 0, 0, NULL }, /* GpioAssertInt */
1104 /* 2*/  EE(ASIC_QSFP1,  handle_qsfp_int,        "QSFP1"),
1105 /* 3*/  EE(ASIC_QSFP2,  handle_qsfp_int,        "QSFP2"),
1106 /* 4*/  { 0, 0, 0, NULL }, /* TCritInt */
1107         /* rest are reserved */
1108 };
1109
1110 /*
1111  * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1112  * register can not be derived from the MTU value because 10K is not
1113  * a power of 2. Therefore, we need a constant. Everything else can
1114  * be calculated.
1115  */
1116 #define DCC_CFG_PORT_MTU_CAP_10240 7
1117
1118 /*
1119  * Table of the DC grouping of error interrupts.  Each entry refers to
1120  * another register containing more information.
1121  */
1122 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1123 /* 0*/  DC_EE1(DCC_ERR,         handle_dcc_err,        "DCC Err"),
1124 /* 1*/  DC_EE2(DC_LCB_ERR,      handle_lcb_err,        "LCB Err"),
1125 /* 2*/  DC_EE2(DC_DC8051_ERR,   handle_8051_interrupt, "DC8051 Interrupt"),
1126 /* 3*/  /* dc_lbm_int - special, see is_dc_int() */
1127         /* the rest are reserved */
1128 };
1129
1130 struct cntr_entry {
1131         /*
1132          * counter name
1133          */
1134         char *name;
1135
1136         /*
1137          * csr to read for name (if applicable)
1138          */
1139         u64 csr;
1140
1141         /*
1142          * offset into dd or ppd to store the counter's value
1143          */
1144         int offset;
1145
1146         /*
1147          * flags
1148          */
1149         u8 flags;
1150
1151         /*
1152          * accessor for stat element, context either dd or ppd
1153          */
1154         u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1155                        int mode, u64 data);
1156 };
1157
1158 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1159 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1160
1161 #define CNTR_ELEM(name, csr, offset, flags, accessor) \
1162 { \
1163         name, \
1164         csr, \
1165         offset, \
1166         flags, \
1167         accessor \
1168 }
1169
1170 /* 32bit RXE */
1171 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1172 CNTR_ELEM(#name, \
1173           (counter * 8 + RCV_COUNTER_ARRAY32), \
1174           0, flags | CNTR_32BIT, \
1175           port_access_u32_csr)
1176
1177 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1178 CNTR_ELEM(#name, \
1179           (counter * 8 + RCV_COUNTER_ARRAY32), \
1180           0, flags | CNTR_32BIT, \
1181           dev_access_u32_csr)
1182
1183 /* 64bit RXE */
1184 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1185 CNTR_ELEM(#name, \
1186           (counter * 8 + RCV_COUNTER_ARRAY64), \
1187           0, flags, \
1188           port_access_u64_csr)
1189
1190 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1191 CNTR_ELEM(#name, \
1192           (counter * 8 + RCV_COUNTER_ARRAY64), \
1193           0, flags, \
1194           dev_access_u64_csr)
1195
1196 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1197 #define OVR_ELM(ctx) \
1198 CNTR_ELEM("RcvHdrOvr" #ctx, \
1199           (RCV_HDR_OVFL_CNT + ctx * 0x100), \
1200           0, CNTR_NORMAL, port_access_u64_csr)
1201
1202 /* 32bit TXE */
1203 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1204 CNTR_ELEM(#name, \
1205           (counter * 8 + SEND_COUNTER_ARRAY32), \
1206           0, flags | CNTR_32BIT, \
1207           port_access_u32_csr)
1208
1209 /* 64bit TXE */
1210 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1211 CNTR_ELEM(#name, \
1212           (counter * 8 + SEND_COUNTER_ARRAY64), \
1213           0, flags, \
1214           port_access_u64_csr)
1215
1216 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1217 CNTR_ELEM(#name,\
1218           counter * 8 + SEND_COUNTER_ARRAY64, \
1219           0, \
1220           flags, \
1221           dev_access_u64_csr)
1222
1223 /* CCE */
1224 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1225 CNTR_ELEM(#name, \
1226           (counter * 8 + CCE_COUNTER_ARRAY32), \
1227           0, flags | CNTR_32BIT, \
1228           dev_access_u32_csr)
1229
1230 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1231 CNTR_ELEM(#name, \
1232           (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1233           0, flags | CNTR_32BIT, \
1234           dev_access_u32_csr)
1235
1236 /* DC */
1237 #define DC_PERF_CNTR(name, counter, flags) \
1238 CNTR_ELEM(#name, \
1239           counter, \
1240           0, \
1241           flags, \
1242           dev_access_u64_csr)
1243
1244 #define DC_PERF_CNTR_LCB(name, counter, flags) \
1245 CNTR_ELEM(#name, \
1246           counter, \
1247           0, \
1248           flags, \
1249           dc_access_lcb_cntr)
1250
1251 /* ibp counters */
1252 #define SW_IBP_CNTR(name, cntr) \
1253 CNTR_ELEM(#name, \
1254           0, \
1255           0, \
1256           CNTR_SYNTH, \
1257           access_ibp_##cntr)
1258
1259 u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1260 {
1261         if (dd->flags & HFI1_PRESENT) {
1262                 return readq((void __iomem *)dd->kregbase + offset);
1263         }
1264         return -1;
1265 }
1266
1267 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1268 {
1269         if (dd->flags & HFI1_PRESENT)
1270                 writeq(value, (void __iomem *)dd->kregbase + offset);
1271 }
1272
1273 void __iomem *get_csr_addr(
1274         struct hfi1_devdata *dd,
1275         u32 offset)
1276 {
1277         return (void __iomem *)dd->kregbase + offset;
1278 }
1279
1280 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1281                                  int mode, u64 value)
1282 {
1283         u64 ret;
1284
1285         if (mode == CNTR_MODE_R) {
1286                 ret = read_csr(dd, csr);
1287         } else if (mode == CNTR_MODE_W) {
1288                 write_csr(dd, csr, value);
1289                 ret = value;
1290         } else {
1291                 dd_dev_err(dd, "Invalid cntr register access mode");
1292                 return 0;
1293         }
1294
1295         hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1296         return ret;
1297 }
1298
1299 /* Dev Access */
1300 static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1301                               void *context, int vl, int mode, u64 data)
1302 {
1303         struct hfi1_devdata *dd = context;
1304         u64 csr = entry->csr;
1305
1306         if (entry->flags & CNTR_SDMA) {
1307                 if (vl == CNTR_INVALID_VL)
1308                         return 0;
1309                 csr += 0x100 * vl;
1310         } else {
1311                 if (vl != CNTR_INVALID_VL)
1312                         return 0;
1313         }
1314         return read_write_csr(dd, csr, mode, data);
1315 }
1316
1317 static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1318                               void *context, int idx, int mode, u64 data)
1319 {
1320         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1321
1322         if (dd->per_sdma && idx < dd->num_sdma)
1323                 return dd->per_sdma[idx].err_cnt;
1324         return 0;
1325 }
1326
1327 static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1328                               void *context, int idx, int mode, u64 data)
1329 {
1330         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1331
1332         if (dd->per_sdma && idx < dd->num_sdma)
1333                 return dd->per_sdma[idx].sdma_int_cnt;
1334         return 0;
1335 }
1336
1337 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1338                                    void *context, int idx, int mode, u64 data)
1339 {
1340         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1341
1342         if (dd->per_sdma && idx < dd->num_sdma)
1343                 return dd->per_sdma[idx].idle_int_cnt;
1344         return 0;
1345 }
1346
1347 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1348                                        void *context, int idx, int mode,
1349                                        u64 data)
1350 {
1351         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1352
1353         if (dd->per_sdma && idx < dd->num_sdma)
1354                 return dd->per_sdma[idx].progress_int_cnt;
1355         return 0;
1356 }
1357
1358 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1359                               int vl, int mode, u64 data)
1360 {
1361         struct hfi1_devdata *dd = context;
1362
1363         u64 val = 0;
1364         u64 csr = entry->csr;
1365
1366         if (entry->flags & CNTR_VL) {
1367                 if (vl == CNTR_INVALID_VL)
1368                         return 0;
1369                 csr += 8 * vl;
1370         } else {
1371                 if (vl != CNTR_INVALID_VL)
1372                         return 0;
1373         }
1374
1375         val = read_write_csr(dd, csr, mode, data);
1376         return val;
1377 }
1378
1379 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1380                               int vl, int mode, u64 data)
1381 {
1382         struct hfi1_devdata *dd = context;
1383         u32 csr = entry->csr;
1384         int ret = 0;
1385
1386         if (vl != CNTR_INVALID_VL)
1387                 return 0;
1388         if (mode == CNTR_MODE_R)
1389                 ret = read_lcb_csr(dd, csr, &data);
1390         else if (mode == CNTR_MODE_W)
1391                 ret = write_lcb_csr(dd, csr, data);
1392
1393         if (ret) {
1394                 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1395                 return 0;
1396         }
1397
1398         hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1399         return data;
1400 }
1401
1402 /* Port Access */
1403 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1404                                int vl, int mode, u64 data)
1405 {
1406         struct hfi1_pportdata *ppd = context;
1407
1408         if (vl != CNTR_INVALID_VL)
1409                 return 0;
1410         return read_write_csr(ppd->dd, entry->csr, mode, data);
1411 }
1412
1413 static u64 port_access_u64_csr(const struct cntr_entry *entry,
1414                                void *context, int vl, int mode, u64 data)
1415 {
1416         struct hfi1_pportdata *ppd = context;
1417         u64 val;
1418         u64 csr = entry->csr;
1419
1420         if (entry->flags & CNTR_VL) {
1421                 if (vl == CNTR_INVALID_VL)
1422                         return 0;
1423                 csr += 8 * vl;
1424         } else {
1425                 if (vl != CNTR_INVALID_VL)
1426                         return 0;
1427         }
1428         val = read_write_csr(ppd->dd, csr, mode, data);
1429         return val;
1430 }
1431
1432 /* Software defined */
1433 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1434                                 u64 data)
1435 {
1436         u64 ret;
1437
1438         if (mode == CNTR_MODE_R) {
1439                 ret = *cntr;
1440         } else if (mode == CNTR_MODE_W) {
1441                 *cntr = data;
1442                 ret = data;
1443         } else {
1444                 dd_dev_err(dd, "Invalid cntr sw access mode");
1445                 return 0;
1446         }
1447
1448         hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1449
1450         return ret;
1451 }
1452
1453 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1454                                  int vl, int mode, u64 data)
1455 {
1456         struct hfi1_pportdata *ppd = context;
1457
1458         if (vl != CNTR_INVALID_VL)
1459                 return 0;
1460         return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1461 }
1462
1463 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1464                                  int vl, int mode, u64 data)
1465 {
1466         struct hfi1_pportdata *ppd = context;
1467
1468         if (vl != CNTR_INVALID_VL)
1469                 return 0;
1470         return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1471 }
1472
1473 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1474                                        void *context, int vl, int mode,
1475                                        u64 data)
1476 {
1477         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1478
1479         if (vl != CNTR_INVALID_VL)
1480                 return 0;
1481         return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1482 }
1483
1484 static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1485                                    void *context, int vl, int mode, u64 data)
1486 {
1487         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1488         u64 zero = 0;
1489         u64 *counter;
1490
1491         if (vl == CNTR_INVALID_VL)
1492                 counter = &ppd->port_xmit_discards;
1493         else if (vl >= 0 && vl < C_VL_COUNT)
1494                 counter = &ppd->port_xmit_discards_vl[vl];
1495         else
1496                 counter = &zero;
1497
1498         return read_write_sw(ppd->dd, counter, mode, data);
1499 }
1500
1501 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1502                                        void *context, int vl, int mode,
1503                                        u64 data)
1504 {
1505         struct hfi1_pportdata *ppd = context;
1506
1507         if (vl != CNTR_INVALID_VL)
1508                 return 0;
1509
1510         return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1511                              mode, data);
1512 }
1513
1514 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1515                                       void *context, int vl, int mode, u64 data)
1516 {
1517         struct hfi1_pportdata *ppd = context;
1518
1519         if (vl != CNTR_INVALID_VL)
1520                 return 0;
1521
1522         return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1523                              mode, data);
1524 }
1525
1526 u64 get_all_cpu_total(u64 __percpu *cntr)
1527 {
1528         int cpu;
1529         u64 counter = 0;
1530
1531         for_each_possible_cpu(cpu)
1532                 counter += *per_cpu_ptr(cntr, cpu);
1533         return counter;
1534 }
1535
1536 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1537                           u64 __percpu *cntr,
1538                           int vl, int mode, u64 data)
1539 {
1540         u64 ret = 0;
1541
1542         if (vl != CNTR_INVALID_VL)
1543                 return 0;
1544
1545         if (mode == CNTR_MODE_R) {
1546                 ret = get_all_cpu_total(cntr) - *z_val;
1547         } else if (mode == CNTR_MODE_W) {
1548                 /* A write can only zero the counter */
1549                 if (data == 0)
1550                         *z_val = get_all_cpu_total(cntr);
1551                 else
1552                         dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1553         } else {
1554                 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1555                 return 0;
1556         }
1557
1558         return ret;
1559 }
1560
1561 static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1562                               void *context, int vl, int mode, u64 data)
1563 {
1564         struct hfi1_devdata *dd = context;
1565
1566         return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1567                               mode, data);
1568 }
1569
1570 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1571                                    void *context, int vl, int mode, u64 data)
1572 {
1573         struct hfi1_devdata *dd = context;
1574
1575         return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1576                               mode, data);
1577 }
1578
1579 static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1580                               void *context, int vl, int mode, u64 data)
1581 {
1582         struct hfi1_devdata *dd = context;
1583
1584         return dd->verbs_dev.n_piowait;
1585 }
1586
1587 static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1588                                void *context, int vl, int mode, u64 data)
1589 {
1590         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1591
1592         return dd->verbs_dev.n_piodrain;
1593 }
1594
1595 static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1596                               void *context, int vl, int mode, u64 data)
1597 {
1598         struct hfi1_devdata *dd = context;
1599
1600         return dd->verbs_dev.n_txwait;
1601 }
1602
1603 static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1604                                void *context, int vl, int mode, u64 data)
1605 {
1606         struct hfi1_devdata *dd = context;
1607
1608         return dd->verbs_dev.n_kmem_wait;
1609 }
1610
1611 static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1612                                    void *context, int vl, int mode, u64 data)
1613 {
1614         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1615
1616         return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1617                               mode, data);
1618 }
1619
1620 /* Software counters for the error status bits within MISC_ERR_STATUS */
1621 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1622                                              void *context, int vl, int mode,
1623                                              u64 data)
1624 {
1625         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1626
1627         return dd->misc_err_status_cnt[12];
1628 }
1629
1630 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1631                                           void *context, int vl, int mode,
1632                                           u64 data)
1633 {
1634         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1635
1636         return dd->misc_err_status_cnt[11];
1637 }
1638
1639 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1640                                                void *context, int vl, int mode,
1641                                                u64 data)
1642 {
1643         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1644
1645         return dd->misc_err_status_cnt[10];
1646 }
1647
1648 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1649                                                  void *context, int vl,
1650                                                  int mode, u64 data)
1651 {
1652         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1653
1654         return dd->misc_err_status_cnt[9];
1655 }
1656
1657 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1658                                            void *context, int vl, int mode,
1659                                            u64 data)
1660 {
1661         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1662
1663         return dd->misc_err_status_cnt[8];
1664 }
1665
1666 static u64 access_misc_efuse_read_bad_addr_err_cnt(
1667                                 const struct cntr_entry *entry,
1668                                 void *context, int vl, int mode, u64 data)
1669 {
1670         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1671
1672         return dd->misc_err_status_cnt[7];
1673 }
1674
1675 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1676                                                 void *context, int vl,
1677                                                 int mode, u64 data)
1678 {
1679         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1680
1681         return dd->misc_err_status_cnt[6];
1682 }
1683
1684 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1685                                               void *context, int vl, int mode,
1686                                               u64 data)
1687 {
1688         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1689
1690         return dd->misc_err_status_cnt[5];
1691 }
1692
1693 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1694                                             void *context, int vl, int mode,
1695                                             u64 data)
1696 {
1697         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1698
1699         return dd->misc_err_status_cnt[4];
1700 }
1701
1702 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1703                                                  void *context, int vl,
1704                                                  int mode, u64 data)
1705 {
1706         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1707
1708         return dd->misc_err_status_cnt[3];
1709 }
1710
1711 static u64 access_misc_csr_write_bad_addr_err_cnt(
1712                                 const struct cntr_entry *entry,
1713                                 void *context, int vl, int mode, u64 data)
1714 {
1715         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1716
1717         return dd->misc_err_status_cnt[2];
1718 }
1719
1720 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1721                                                  void *context, int vl,
1722                                                  int mode, u64 data)
1723 {
1724         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1725
1726         return dd->misc_err_status_cnt[1];
1727 }
1728
1729 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1730                                           void *context, int vl, int mode,
1731                                           u64 data)
1732 {
1733         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1734
1735         return dd->misc_err_status_cnt[0];
1736 }
1737
1738 /*
1739  * Software counter for the aggregate of
1740  * individual CceErrStatus counters
1741  */
1742 static u64 access_sw_cce_err_status_aggregated_cnt(
1743                                 const struct cntr_entry *entry,
1744                                 void *context, int vl, int mode, u64 data)
1745 {
1746         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1747
1748         return dd->sw_cce_err_status_aggregate;
1749 }
1750
1751 /*
1752  * Software counters corresponding to each of the
1753  * error status bits within CceErrStatus
1754  */
1755 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1756                                               void *context, int vl, int mode,
1757                                               u64 data)
1758 {
1759         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1760
1761         return dd->cce_err_status_cnt[40];
1762 }
1763
1764 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1765                                           void *context, int vl, int mode,
1766                                           u64 data)
1767 {
1768         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1769
1770         return dd->cce_err_status_cnt[39];
1771 }
1772
1773 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1774                                           void *context, int vl, int mode,
1775                                           u64 data)
1776 {
1777         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1778
1779         return dd->cce_err_status_cnt[38];
1780 }
1781
1782 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1783                                              void *context, int vl, int mode,
1784                                              u64 data)
1785 {
1786         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1787
1788         return dd->cce_err_status_cnt[37];
1789 }
1790
1791 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1792                                              void *context, int vl, int mode,
1793                                              u64 data)
1794 {
1795         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1796
1797         return dd->cce_err_status_cnt[36];
1798 }
1799
1800 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1801                                 const struct cntr_entry *entry,
1802                                 void *context, int vl, int mode, u64 data)
1803 {
1804         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1805
1806         return dd->cce_err_status_cnt[35];
1807 }
1808
1809 static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1810                                 const struct cntr_entry *entry,
1811                                 void *context, int vl, int mode, u64 data)
1812 {
1813         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1814
1815         return dd->cce_err_status_cnt[34];
1816 }
1817
1818 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1819                                                  void *context, int vl,
1820                                                  int mode, u64 data)
1821 {
1822         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1823
1824         return dd->cce_err_status_cnt[33];
1825 }
1826
1827 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1828                                                 void *context, int vl, int mode,
1829                                                 u64 data)
1830 {
1831         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1832
1833         return dd->cce_err_status_cnt[32];
1834 }
1835
1836 static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1837                                    void *context, int vl, int mode, u64 data)
1838 {
1839         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1840
1841         return dd->cce_err_status_cnt[31];
1842 }
1843
1844 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1845                                                void *context, int vl, int mode,
1846                                                u64 data)
1847 {
1848         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1849
1850         return dd->cce_err_status_cnt[30];
1851 }
1852
1853 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1854                                               void *context, int vl, int mode,
1855                                               u64 data)
1856 {
1857         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1858
1859         return dd->cce_err_status_cnt[29];
1860 }
1861
1862 static u64 access_pcic_transmit_back_parity_err_cnt(
1863                                 const struct cntr_entry *entry,
1864                                 void *context, int vl, int mode, u64 data)
1865 {
1866         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1867
1868         return dd->cce_err_status_cnt[28];
1869 }
1870
1871 static u64 access_pcic_transmit_front_parity_err_cnt(
1872                                 const struct cntr_entry *entry,
1873                                 void *context, int vl, int mode, u64 data)
1874 {
1875         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1876
1877         return dd->cce_err_status_cnt[27];
1878 }
1879
1880 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1881                                              void *context, int vl, int mode,
1882                                              u64 data)
1883 {
1884         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1885
1886         return dd->cce_err_status_cnt[26];
1887 }
1888
1889 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1890                                             void *context, int vl, int mode,
1891                                             u64 data)
1892 {
1893         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1894
1895         return dd->cce_err_status_cnt[25];
1896 }
1897
1898 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1899                                               void *context, int vl, int mode,
1900                                               u64 data)
1901 {
1902         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1903
1904         return dd->cce_err_status_cnt[24];
1905 }
1906
1907 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1908                                              void *context, int vl, int mode,
1909                                              u64 data)
1910 {
1911         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1912
1913         return dd->cce_err_status_cnt[23];
1914 }
1915
1916 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
1917                                                  void *context, int vl,
1918                                                  int mode, u64 data)
1919 {
1920         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1921
1922         return dd->cce_err_status_cnt[22];
1923 }
1924
1925 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
1926                                          void *context, int vl, int mode,
1927                                          u64 data)
1928 {
1929         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1930
1931         return dd->cce_err_status_cnt[21];
1932 }
1933
1934 static u64 access_pcic_n_post_dat_q_parity_err_cnt(
1935                                 const struct cntr_entry *entry,
1936                                 void *context, int vl, int mode, u64 data)
1937 {
1938         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1939
1940         return dd->cce_err_status_cnt[20];
1941 }
1942
1943 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
1944                                                  void *context, int vl,
1945                                                  int mode, u64 data)
1946 {
1947         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1948
1949         return dd->cce_err_status_cnt[19];
1950 }
1951
1952 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1953                                              void *context, int vl, int mode,
1954                                              u64 data)
1955 {
1956         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1957
1958         return dd->cce_err_status_cnt[18];
1959 }
1960
1961 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1962                                             void *context, int vl, int mode,
1963                                             u64 data)
1964 {
1965         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1966
1967         return dd->cce_err_status_cnt[17];
1968 }
1969
1970 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
1971                                               void *context, int vl, int mode,
1972                                               u64 data)
1973 {
1974         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1975
1976         return dd->cce_err_status_cnt[16];
1977 }
1978
1979 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
1980                                              void *context, int vl, int mode,
1981                                              u64 data)
1982 {
1983         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1984
1985         return dd->cce_err_status_cnt[15];
1986 }
1987
1988 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
1989                                                  void *context, int vl,
1990                                                  int mode, u64 data)
1991 {
1992         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1993
1994         return dd->cce_err_status_cnt[14];
1995 }
1996
1997 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
1998                                              void *context, int vl, int mode,
1999                                              u64 data)
2000 {
2001         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2002
2003         return dd->cce_err_status_cnt[13];
2004 }
2005
2006 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2007                                 const struct cntr_entry *entry,
2008                                 void *context, int vl, int mode, u64 data)
2009 {
2010         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2011
2012         return dd->cce_err_status_cnt[12];
2013 }
2014
2015 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2016                                 const struct cntr_entry *entry,
2017                                 void *context, int vl, int mode, u64 data)
2018 {
2019         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2020
2021         return dd->cce_err_status_cnt[11];
2022 }
2023
2024 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2025                                 const struct cntr_entry *entry,
2026                                 void *context, int vl, int mode, u64 data)
2027 {
2028         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2029
2030         return dd->cce_err_status_cnt[10];
2031 }
2032
2033 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2034                                 const struct cntr_entry *entry,
2035                                 void *context, int vl, int mode, u64 data)
2036 {
2037         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2038
2039         return dd->cce_err_status_cnt[9];
2040 }
2041
2042 static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2043                                 const struct cntr_entry *entry,
2044                                 void *context, int vl, int mode, u64 data)
2045 {
2046         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2047
2048         return dd->cce_err_status_cnt[8];
2049 }
2050
2051 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2052                                                  void *context, int vl,
2053                                                  int mode, u64 data)
2054 {
2055         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2056
2057         return dd->cce_err_status_cnt[7];
2058 }
2059
2060 static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2061                                 const struct cntr_entry *entry,
2062                                 void *context, int vl, int mode, u64 data)
2063 {
2064         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2065
2066         return dd->cce_err_status_cnt[6];
2067 }
2068
2069 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2070                                                void *context, int vl, int mode,
2071                                                u64 data)
2072 {
2073         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2074
2075         return dd->cce_err_status_cnt[5];
2076 }
2077
2078 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2079                                           void *context, int vl, int mode,
2080                                           u64 data)
2081 {
2082         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2083
2084         return dd->cce_err_status_cnt[4];
2085 }
2086
2087 static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2088                                 const struct cntr_entry *entry,
2089                                 void *context, int vl, int mode, u64 data)
2090 {
2091         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2092
2093         return dd->cce_err_status_cnt[3];
2094 }
2095
2096 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2097                                                  void *context, int vl,
2098                                                  int mode, u64 data)
2099 {
2100         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2101
2102         return dd->cce_err_status_cnt[2];
2103 }
2104
2105 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2106                                                 void *context, int vl,
2107                                                 int mode, u64 data)
2108 {
2109         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2110
2111         return dd->cce_err_status_cnt[1];
2112 }
2113
2114 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2115                                          void *context, int vl, int mode,
2116                                          u64 data)
2117 {
2118         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2119
2120         return dd->cce_err_status_cnt[0];
2121 }
2122
2123 /*
2124  * Software counters corresponding to each of the
2125  * error status bits within RcvErrStatus
2126  */
2127 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2128                                         void *context, int vl, int mode,
2129                                         u64 data)
2130 {
2131         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2132
2133         return dd->rcv_err_status_cnt[63];
2134 }
2135
2136 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2137                                                 void *context, int vl,
2138                                                 int mode, u64 data)
2139 {
2140         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2141
2142         return dd->rcv_err_status_cnt[62];
2143 }
2144
2145 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2146                                                void *context, int vl, int mode,
2147                                                u64 data)
2148 {
2149         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2150
2151         return dd->rcv_err_status_cnt[61];
2152 }
2153
2154 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2155                                          void *context, int vl, int mode,
2156                                          u64 data)
2157 {
2158         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2159
2160         return dd->rcv_err_status_cnt[60];
2161 }
2162
2163 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2164                                                  void *context, int vl,
2165                                                  int mode, u64 data)
2166 {
2167         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2168
2169         return dd->rcv_err_status_cnt[59];
2170 }
2171
2172 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2173                                                  void *context, int vl,
2174                                                  int mode, u64 data)
2175 {
2176         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2177
2178         return dd->rcv_err_status_cnt[58];
2179 }
2180
2181 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2182                                             void *context, int vl, int mode,
2183                                             u64 data)
2184 {
2185         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2186
2187         return dd->rcv_err_status_cnt[57];
2188 }
2189
2190 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2191                                            void *context, int vl, int mode,
2192                                            u64 data)
2193 {
2194         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2195
2196         return dd->rcv_err_status_cnt[56];
2197 }
2198
2199 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2200                                            void *context, int vl, int mode,
2201                                            u64 data)
2202 {
2203         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2204
2205         return dd->rcv_err_status_cnt[55];
2206 }
2207
2208 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2209                                 const struct cntr_entry *entry,
2210                                 void *context, int vl, int mode, u64 data)
2211 {
2212         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2213
2214         return dd->rcv_err_status_cnt[54];
2215 }
2216
2217 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2218                                 const struct cntr_entry *entry,
2219                                 void *context, int vl, int mode, u64 data)
2220 {
2221         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2222
2223         return dd->rcv_err_status_cnt[53];
2224 }
2225
2226 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2227                                                  void *context, int vl,
2228                                                  int mode, u64 data)
2229 {
2230         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2231
2232         return dd->rcv_err_status_cnt[52];
2233 }
2234
2235 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2236                                                  void *context, int vl,
2237                                                  int mode, u64 data)
2238 {
2239         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2240
2241         return dd->rcv_err_status_cnt[51];
2242 }
2243
2244 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2245                                                  void *context, int vl,
2246                                                  int mode, u64 data)
2247 {
2248         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2249
2250         return dd->rcv_err_status_cnt[50];
2251 }
2252
2253 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2254                                                  void *context, int vl,
2255                                                  int mode, u64 data)
2256 {
2257         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2258
2259         return dd->rcv_err_status_cnt[49];
2260 }
2261
2262 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2263                                                  void *context, int vl,
2264                                                  int mode, u64 data)
2265 {
2266         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2267
2268         return dd->rcv_err_status_cnt[48];
2269 }
2270
2271 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2272                                                  void *context, int vl,
2273                                                  int mode, u64 data)
2274 {
2275         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2276
2277         return dd->rcv_err_status_cnt[47];
2278 }
2279
2280 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2281                                          void *context, int vl, int mode,
2282                                          u64 data)
2283 {
2284         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2285
2286         return dd->rcv_err_status_cnt[46];
2287 }
2288
2289 static u64 access_rx_hq_intr_csr_parity_err_cnt(
2290                                 const struct cntr_entry *entry,
2291                                 void *context, int vl, int mode, u64 data)
2292 {
2293         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2294
2295         return dd->rcv_err_status_cnt[45];
2296 }
2297
2298 static u64 access_rx_lookup_csr_parity_err_cnt(
2299                                 const struct cntr_entry *entry,
2300                                 void *context, int vl, int mode, u64 data)
2301 {
2302         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2303
2304         return dd->rcv_err_status_cnt[44];
2305 }
2306
2307 static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2308                                 const struct cntr_entry *entry,
2309                                 void *context, int vl, int mode, u64 data)
2310 {
2311         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2312
2313         return dd->rcv_err_status_cnt[43];
2314 }
2315
2316 static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2317                                 const struct cntr_entry *entry,
2318                                 void *context, int vl, int mode, u64 data)
2319 {
2320         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2321
2322         return dd->rcv_err_status_cnt[42];
2323 }
2324
2325 static u64 access_rx_lookup_des_part2_parity_err_cnt(
2326                                 const struct cntr_entry *entry,
2327                                 void *context, int vl, int mode, u64 data)
2328 {
2329         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2330
2331         return dd->rcv_err_status_cnt[41];
2332 }
2333
2334 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2335                                 const struct cntr_entry *entry,
2336                                 void *context, int vl, int mode, u64 data)
2337 {
2338         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2339
2340         return dd->rcv_err_status_cnt[40];
2341 }
2342
2343 static u64 access_rx_lookup_des_part1_unc_err_cnt(
2344                                 const struct cntr_entry *entry,
2345                                 void *context, int vl, int mode, u64 data)
2346 {
2347         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2348
2349         return dd->rcv_err_status_cnt[39];
2350 }
2351
2352 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2353                                 const struct cntr_entry *entry,
2354                                 void *context, int vl, int mode, u64 data)
2355 {
2356         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2357
2358         return dd->rcv_err_status_cnt[38];
2359 }
2360
2361 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2362                                 const struct cntr_entry *entry,
2363                                 void *context, int vl, int mode, u64 data)
2364 {
2365         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2366
2367         return dd->rcv_err_status_cnt[37];
2368 }
2369
2370 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2371                                 const struct cntr_entry *entry,
2372                                 void *context, int vl, int mode, u64 data)
2373 {
2374         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2375
2376         return dd->rcv_err_status_cnt[36];
2377 }
2378
2379 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2380                                 const struct cntr_entry *entry,
2381                                 void *context, int vl, int mode, u64 data)
2382 {
2383         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2384
2385         return dd->rcv_err_status_cnt[35];
2386 }
2387
2388 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2389                                 const struct cntr_entry *entry,
2390                                 void *context, int vl, int mode, u64 data)
2391 {
2392         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2393
2394         return dd->rcv_err_status_cnt[34];
2395 }
2396
2397 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2398                                 const struct cntr_entry *entry,
2399                                 void *context, int vl, int mode, u64 data)
2400 {
2401         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2402
2403         return dd->rcv_err_status_cnt[33];
2404 }
2405
2406 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2407                                         void *context, int vl, int mode,
2408                                         u64 data)
2409 {
2410         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2411
2412         return dd->rcv_err_status_cnt[32];
2413 }
2414
2415 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2416                                        void *context, int vl, int mode,
2417                                        u64 data)
2418 {
2419         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2420
2421         return dd->rcv_err_status_cnt[31];
2422 }
2423
2424 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2425                                           void *context, int vl, int mode,
2426                                           u64 data)
2427 {
2428         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2429
2430         return dd->rcv_err_status_cnt[30];
2431 }
2432
2433 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2434                                              void *context, int vl, int mode,
2435                                              u64 data)
2436 {
2437         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2438
2439         return dd->rcv_err_status_cnt[29];
2440 }
2441
2442 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2443                                                  void *context, int vl,
2444                                                  int mode, u64 data)
2445 {
2446         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2447
2448         return dd->rcv_err_status_cnt[28];
2449 }
2450
2451 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2452                                 const struct cntr_entry *entry,
2453                                 void *context, int vl, int mode, u64 data)
2454 {
2455         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2456
2457         return dd->rcv_err_status_cnt[27];
2458 }
2459
2460 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2461                                 const struct cntr_entry *entry,
2462                                 void *context, int vl, int mode, u64 data)
2463 {
2464         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2465
2466         return dd->rcv_err_status_cnt[26];
2467 }
2468
2469 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2470                                 const struct cntr_entry *entry,
2471                                 void *context, int vl, int mode, u64 data)
2472 {
2473         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2474
2475         return dd->rcv_err_status_cnt[25];
2476 }
2477
2478 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2479                                 const struct cntr_entry *entry,
2480                                 void *context, int vl, int mode, u64 data)
2481 {
2482         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2483
2484         return dd->rcv_err_status_cnt[24];
2485 }
2486
2487 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2488                                 const struct cntr_entry *entry,
2489                                 void *context, int vl, int mode, u64 data)
2490 {
2491         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2492
2493         return dd->rcv_err_status_cnt[23];
2494 }
2495
2496 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2497                                 const struct cntr_entry *entry,
2498                                 void *context, int vl, int mode, u64 data)
2499 {
2500         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2501
2502         return dd->rcv_err_status_cnt[22];
2503 }
2504
2505 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2506                                 const struct cntr_entry *entry,
2507                                 void *context, int vl, int mode, u64 data)
2508 {
2509         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2510
2511         return dd->rcv_err_status_cnt[21];
2512 }
2513
2514 static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2515                                 const struct cntr_entry *entry,
2516                                 void *context, int vl, int mode, u64 data)
2517 {
2518         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2519
2520         return dd->rcv_err_status_cnt[20];
2521 }
2522
2523 static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2524                                 const struct cntr_entry *entry,
2525                                 void *context, int vl, int mode, u64 data)
2526 {
2527         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2528
2529         return dd->rcv_err_status_cnt[19];
2530 }
2531
2532 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2533                                                  void *context, int vl,
2534                                                  int mode, u64 data)
2535 {
2536         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2537
2538         return dd->rcv_err_status_cnt[18];
2539 }
2540
2541 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2542                                                  void *context, int vl,
2543                                                  int mode, u64 data)
2544 {
2545         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2546
2547         return dd->rcv_err_status_cnt[17];
2548 }
2549
2550 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2551                                 const struct cntr_entry *entry,
2552                                 void *context, int vl, int mode, u64 data)
2553 {
2554         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2555
2556         return dd->rcv_err_status_cnt[16];
2557 }
2558
2559 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2560                                 const struct cntr_entry *entry,
2561                                 void *context, int vl, int mode, u64 data)
2562 {
2563         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2564
2565         return dd->rcv_err_status_cnt[15];
2566 }
2567
2568 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2569                                                 void *context, int vl,
2570                                                 int mode, u64 data)
2571 {
2572         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2573
2574         return dd->rcv_err_status_cnt[14];
2575 }
2576
2577 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2578                                                 void *context, int vl,
2579                                                 int mode, u64 data)
2580 {
2581         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2582
2583         return dd->rcv_err_status_cnt[13];
2584 }
2585
2586 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2587                                               void *context, int vl, int mode,
2588                                               u64 data)
2589 {
2590         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2591
2592         return dd->rcv_err_status_cnt[12];
2593 }
2594
2595 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2596                                           void *context, int vl, int mode,
2597                                           u64 data)
2598 {
2599         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2600
2601         return dd->rcv_err_status_cnt[11];
2602 }
2603
2604 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2605                                           void *context, int vl, int mode,
2606                                           u64 data)
2607 {
2608         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2609
2610         return dd->rcv_err_status_cnt[10];
2611 }
2612
2613 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2614                                                void *context, int vl, int mode,
2615                                                u64 data)
2616 {
2617         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2618
2619         return dd->rcv_err_status_cnt[9];
2620 }
2621
2622 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2623                                             void *context, int vl, int mode,
2624                                             u64 data)
2625 {
2626         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2627
2628         return dd->rcv_err_status_cnt[8];
2629 }
2630
2631 static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2632                                 const struct cntr_entry *entry,
2633                                 void *context, int vl, int mode, u64 data)
2634 {
2635         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2636
2637         return dd->rcv_err_status_cnt[7];
2638 }
2639
2640 static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2641                                 const struct cntr_entry *entry,
2642                                 void *context, int vl, int mode, u64 data)
2643 {
2644         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2645
2646         return dd->rcv_err_status_cnt[6];
2647 }
2648
2649 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2650                                           void *context, int vl, int mode,
2651                                           u64 data)
2652 {
2653         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2654
2655         return dd->rcv_err_status_cnt[5];
2656 }
2657
2658 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2659                                           void *context, int vl, int mode,
2660                                           u64 data)
2661 {
2662         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2663
2664         return dd->rcv_err_status_cnt[4];
2665 }
2666
2667 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2668                                          void *context, int vl, int mode,
2669                                          u64 data)
2670 {
2671         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2672
2673         return dd->rcv_err_status_cnt[3];
2674 }
2675
2676 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2677                                          void *context, int vl, int mode,
2678                                          u64 data)
2679 {
2680         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2681
2682         return dd->rcv_err_status_cnt[2];
2683 }
2684
2685 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2686                                             void *context, int vl, int mode,
2687                                             u64 data)
2688 {
2689         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2690
2691         return dd->rcv_err_status_cnt[1];
2692 }
2693
2694 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2695                                          void *context, int vl, int mode,
2696                                          u64 data)
2697 {
2698         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2699
2700         return dd->rcv_err_status_cnt[0];
2701 }
2702
2703 /*
2704  * Software counters corresponding to each of the
2705  * error status bits within SendPioErrStatus
2706  */
2707 static u64 access_pio_pec_sop_head_parity_err_cnt(
2708                                 const struct cntr_entry *entry,
2709                                 void *context, int vl, int mode, u64 data)
2710 {
2711         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2712
2713         return dd->send_pio_err_status_cnt[35];
2714 }
2715
2716 static u64 access_pio_pcc_sop_head_parity_err_cnt(
2717                                 const struct cntr_entry *entry,
2718                                 void *context, int vl, int mode, u64 data)
2719 {
2720         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2721
2722         return dd->send_pio_err_status_cnt[34];
2723 }
2724
2725 static u64 access_pio_last_returned_cnt_parity_err_cnt(
2726                                 const struct cntr_entry *entry,
2727                                 void *context, int vl, int mode, u64 data)
2728 {
2729         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2730
2731         return dd->send_pio_err_status_cnt[33];
2732 }
2733
2734 static u64 access_pio_current_free_cnt_parity_err_cnt(
2735                                 const struct cntr_entry *entry,
2736                                 void *context, int vl, int mode, u64 data)
2737 {
2738         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2739
2740         return dd->send_pio_err_status_cnt[32];
2741 }
2742
2743 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2744                                           void *context, int vl, int mode,
2745                                           u64 data)
2746 {
2747         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2748
2749         return dd->send_pio_err_status_cnt[31];
2750 }
2751
2752 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2753                                           void *context, int vl, int mode,
2754                                           u64 data)
2755 {
2756         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2757
2758         return dd->send_pio_err_status_cnt[30];
2759 }
2760
2761 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2762                                            void *context, int vl, int mode,
2763                                            u64 data)
2764 {
2765         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2766
2767         return dd->send_pio_err_status_cnt[29];
2768 }
2769
2770 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2771                                 const struct cntr_entry *entry,
2772                                 void *context, int vl, int mode, u64 data)
2773 {
2774         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2775
2776         return dd->send_pio_err_status_cnt[28];
2777 }
2778
2779 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2780                                              void *context, int vl, int mode,
2781                                              u64 data)
2782 {
2783         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2784
2785         return dd->send_pio_err_status_cnt[27];
2786 }
2787
2788 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2789                                              void *context, int vl, int mode,
2790                                              u64 data)
2791 {
2792         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2793
2794         return dd->send_pio_err_status_cnt[26];
2795 }
2796
2797 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2798                                                 void *context, int vl,
2799                                                 int mode, u64 data)
2800 {
2801         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2802
2803         return dd->send_pio_err_status_cnt[25];
2804 }
2805
2806 static u64 access_pio_block_qw_count_parity_err_cnt(
2807                                 const struct cntr_entry *entry,
2808                                 void *context, int vl, int mode, u64 data)
2809 {
2810         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2811
2812         return dd->send_pio_err_status_cnt[24];
2813 }
2814
2815 static u64 access_pio_write_qw_valid_parity_err_cnt(
2816                                 const struct cntr_entry *entry,
2817                                 void *context, int vl, int mode, u64 data)
2818 {
2819         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2820
2821         return dd->send_pio_err_status_cnt[23];
2822 }
2823
2824 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2825                                             void *context, int vl, int mode,
2826                                             u64 data)
2827 {
2828         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2829
2830         return dd->send_pio_err_status_cnt[22];
2831 }
2832
2833 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2834                                                 void *context, int vl,
2835                                                 int mode, u64 data)
2836 {
2837         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2838
2839         return dd->send_pio_err_status_cnt[21];
2840 }
2841
2842 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2843                                                 void *context, int vl,
2844                                                 int mode, u64 data)
2845 {
2846         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2847
2848         return dd->send_pio_err_status_cnt[20];
2849 }
2850
2851 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2852                                                 void *context, int vl,
2853                                                 int mode, u64 data)
2854 {
2855         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2856
2857         return dd->send_pio_err_status_cnt[19];
2858 }
2859
2860 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2861                                 const struct cntr_entry *entry,
2862                                 void *context, int vl, int mode, u64 data)
2863 {
2864         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2865
2866         return dd->send_pio_err_status_cnt[18];
2867 }
2868
2869 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2870                                          void *context, int vl, int mode,
2871                                          u64 data)
2872 {
2873         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2874
2875         return dd->send_pio_err_status_cnt[17];
2876 }
2877
2878 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2879                                             void *context, int vl, int mode,
2880                                             u64 data)
2881 {
2882         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2883
2884         return dd->send_pio_err_status_cnt[16];
2885 }
2886
2887 static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2888                                 const struct cntr_entry *entry,
2889                                 void *context, int vl, int mode, u64 data)
2890 {
2891         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2892
2893         return dd->send_pio_err_status_cnt[15];
2894 }
2895
2896 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2897                                 const struct cntr_entry *entry,
2898                                 void *context, int vl, int mode, u64 data)
2899 {
2900         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2901
2902         return dd->send_pio_err_status_cnt[14];
2903 }
2904
2905 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2906                                 const struct cntr_entry *entry,
2907                                 void *context, int vl, int mode, u64 data)
2908 {
2909         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2910
2911         return dd->send_pio_err_status_cnt[13];
2912 }
2913
2914 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
2915                                 const struct cntr_entry *entry,
2916                                 void *context, int vl, int mode, u64 data)
2917 {
2918         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2919
2920         return dd->send_pio_err_status_cnt[12];
2921 }
2922
2923 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
2924                                 const struct cntr_entry *entry,
2925                                 void *context, int vl, int mode, u64 data)
2926 {
2927         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2928
2929         return dd->send_pio_err_status_cnt[11];
2930 }
2931
2932 static u64 access_pio_sm_pkt_reset_parity_err_cnt(
2933                                 const struct cntr_entry *entry,
2934                                 void *context, int vl, int mode, u64 data)
2935 {
2936         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2937
2938         return dd->send_pio_err_status_cnt[10];
2939 }
2940
2941 static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
2942                                 const struct cntr_entry *entry,
2943                                 void *context, int vl, int mode, u64 data)
2944 {
2945         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2946
2947         return dd->send_pio_err_status_cnt[9];
2948 }
2949
2950 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
2951                                 const struct cntr_entry *entry,
2952                                 void *context, int vl, int mode, u64 data)
2953 {
2954         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2955
2956         return dd->send_pio_err_status_cnt[8];
2957 }
2958
2959 static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
2960                                 const struct cntr_entry *entry,
2961                                 void *context, int vl, int mode, u64 data)
2962 {
2963         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2964
2965         return dd->send_pio_err_status_cnt[7];
2966 }
2967
2968 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
2969                                               void *context, int vl, int mode,
2970                                               u64 data)
2971 {
2972         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2973
2974         return dd->send_pio_err_status_cnt[6];
2975 }
2976
2977 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
2978                                               void *context, int vl, int mode,
2979                                               u64 data)
2980 {
2981         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2982
2983         return dd->send_pio_err_status_cnt[5];
2984 }
2985
2986 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
2987                                            void *context, int vl, int mode,
2988                                            u64 data)
2989 {
2990         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2991
2992         return dd->send_pio_err_status_cnt[4];
2993 }
2994
2995 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
2996                                            void *context, int vl, int mode,
2997                                            u64 data)
2998 {
2999         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3000
3001         return dd->send_pio_err_status_cnt[3];
3002 }
3003
3004 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3005                                          void *context, int vl, int mode,
3006                                          u64 data)
3007 {
3008         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3009
3010         return dd->send_pio_err_status_cnt[2];
3011 }
3012
3013 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3014                                                 void *context, int vl,
3015                                                 int mode, u64 data)
3016 {
3017         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3018
3019         return dd->send_pio_err_status_cnt[1];
3020 }
3021
3022 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3023                                              void *context, int vl, int mode,
3024                                              u64 data)
3025 {
3026         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3027
3028         return dd->send_pio_err_status_cnt[0];
3029 }
3030
3031 /*
3032  * Software counters corresponding to each of the
3033  * error status bits within SendDmaErrStatus
3034  */
3035 static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3036                                 const struct cntr_entry *entry,
3037                                 void *context, int vl, int mode, u64 data)
3038 {
3039         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3040
3041         return dd->send_dma_err_status_cnt[3];
3042 }
3043
3044 static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3045                                 const struct cntr_entry *entry,
3046                                 void *context, int vl, int mode, u64 data)
3047 {
3048         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3049
3050         return dd->send_dma_err_status_cnt[2];
3051 }
3052
3053 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3054                                           void *context, int vl, int mode,
3055                                           u64 data)
3056 {
3057         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3058
3059         return dd->send_dma_err_status_cnt[1];
3060 }
3061
3062 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3063                                        void *context, int vl, int mode,
3064                                        u64 data)
3065 {
3066         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3067
3068         return dd->send_dma_err_status_cnt[0];
3069 }
3070
3071 /*
3072  * Software counters corresponding to each of the
3073  * error status bits within SendEgressErrStatus
3074  */
3075 static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3076                                 const struct cntr_entry *entry,
3077                                 void *context, int vl, int mode, u64 data)
3078 {
3079         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3080
3081         return dd->send_egress_err_status_cnt[63];
3082 }
3083
3084 static u64 access_tx_read_sdma_memory_csr_err_cnt(
3085                                 const struct cntr_entry *entry,
3086                                 void *context, int vl, int mode, u64 data)
3087 {
3088         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3089
3090         return dd->send_egress_err_status_cnt[62];
3091 }
3092
3093 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3094                                              void *context, int vl, int mode,
3095                                              u64 data)
3096 {
3097         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3098
3099         return dd->send_egress_err_status_cnt[61];
3100 }
3101
3102 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3103                                                  void *context, int vl,
3104                                                  int mode, u64 data)
3105 {
3106         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3107
3108         return dd->send_egress_err_status_cnt[60];
3109 }
3110
3111 static u64 access_tx_read_sdma_memory_cor_err_cnt(
3112                                 const struct cntr_entry *entry,
3113                                 void *context, int vl, int mode, u64 data)
3114 {
3115         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3116
3117         return dd->send_egress_err_status_cnt[59];
3118 }
3119
3120 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3121                                         void *context, int vl, int mode,
3122                                         u64 data)
3123 {
3124         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3125
3126         return dd->send_egress_err_status_cnt[58];
3127 }
3128
3129 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3130                                             void *context, int vl, int mode,
3131                                             u64 data)
3132 {
3133         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3134
3135         return dd->send_egress_err_status_cnt[57];
3136 }
3137
3138 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3139                                               void *context, int vl, int mode,
3140                                               u64 data)
3141 {
3142         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3143
3144         return dd->send_egress_err_status_cnt[56];
3145 }
3146
3147 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3148                                               void *context, int vl, int mode,
3149                                               u64 data)
3150 {
3151         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3152
3153         return dd->send_egress_err_status_cnt[55];
3154 }
3155
3156 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3157                                               void *context, int vl, int mode,
3158                                               u64 data)
3159 {
3160         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3161
3162         return dd->send_egress_err_status_cnt[54];
3163 }
3164
3165 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3166                                               void *context, int vl, int mode,
3167                                               u64 data)
3168 {
3169         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3170
3171         return dd->send_egress_err_status_cnt[53];
3172 }
3173
3174 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3175                                               void *context, int vl, int mode,
3176                                               u64 data)
3177 {
3178         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3179
3180         return dd->send_egress_err_status_cnt[52];
3181 }
3182
3183 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3184                                               void *context, int vl, int mode,
3185                                               u64 data)
3186 {
3187         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3188
3189         return dd->send_egress_err_status_cnt[51];
3190 }
3191
3192 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3193                                               void *context, int vl, int mode,
3194                                               u64 data)
3195 {
3196         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3197
3198         return dd->send_egress_err_status_cnt[50];
3199 }
3200
3201 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3202                                               void *context, int vl, int mode,
3203                                               u64 data)
3204 {
3205         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3206
3207         return dd->send_egress_err_status_cnt[49];
3208 }
3209
3210 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3211                                               void *context, int vl, int mode,
3212                                               u64 data)
3213 {
3214         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3215
3216         return dd->send_egress_err_status_cnt[48];
3217 }
3218
3219 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3220                                               void *context, int vl, int mode,
3221                                               u64 data)
3222 {
3223         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3224
3225         return dd->send_egress_err_status_cnt[47];
3226 }
3227
3228 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3229                                             void *context, int vl, int mode,
3230                                             u64 data)
3231 {
3232         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3233
3234         return dd->send_egress_err_status_cnt[46];
3235 }
3236
3237 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3238                                              void *context, int vl, int mode,
3239                                              u64 data)
3240 {
3241         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3242
3243         return dd->send_egress_err_status_cnt[45];
3244 }
3245
3246 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3247                                                  void *context, int vl,
3248                                                  int mode, u64 data)
3249 {
3250         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3251
3252         return dd->send_egress_err_status_cnt[44];
3253 }
3254
3255 static u64 access_tx_read_sdma_memory_unc_err_cnt(
3256                                 const struct cntr_entry *entry,
3257                                 void *context, int vl, int mode, u64 data)
3258 {
3259         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3260
3261         return dd->send_egress_err_status_cnt[43];
3262 }
3263
3264 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3265                                         void *context, int vl, int mode,
3266                                         u64 data)
3267 {
3268         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3269
3270         return dd->send_egress_err_status_cnt[42];
3271 }
3272
3273 static u64 access_tx_credit_return_partiy_err_cnt(
3274                                 const struct cntr_entry *entry,
3275                                 void *context, int vl, int mode, u64 data)
3276 {
3277         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3278
3279         return dd->send_egress_err_status_cnt[41];
3280 }
3281
3282 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3283                                 const struct cntr_entry *entry,
3284                                 void *context, int vl, int mode, u64 data)
3285 {
3286         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3287
3288         return dd->send_egress_err_status_cnt[40];
3289 }
3290
3291 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3292                                 const struct cntr_entry *entry,
3293                                 void *context, int vl, int mode, u64 data)
3294 {
3295         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3296
3297         return dd->send_egress_err_status_cnt[39];
3298 }
3299
3300 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3301                                 const struct cntr_entry *entry,
3302                                 void *context, int vl, int mode, u64 data)
3303 {
3304         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3305
3306         return dd->send_egress_err_status_cnt[38];
3307 }
3308
3309 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3310                                 const struct cntr_entry *entry,
3311                                 void *context, int vl, int mode, u64 data)
3312 {
3313         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3314
3315         return dd->send_egress_err_status_cnt[37];
3316 }
3317
3318 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3319                                 const struct cntr_entry *entry,
3320                                 void *context, int vl, int mode, u64 data)
3321 {
3322         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3323
3324         return dd->send_egress_err_status_cnt[36];
3325 }
3326
3327 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3328                                 const struct cntr_entry *entry,
3329                                 void *context, int vl, int mode, u64 data)
3330 {
3331         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3332
3333         return dd->send_egress_err_status_cnt[35];
3334 }
3335
3336 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3337                                 const struct cntr_entry *entry,
3338                                 void *context, int vl, int mode, u64 data)
3339 {
3340         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3341
3342         return dd->send_egress_err_status_cnt[34];
3343 }
3344
3345 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3346                                 const struct cntr_entry *entry,
3347                                 void *context, int vl, int mode, u64 data)
3348 {
3349         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3350
3351         return dd->send_egress_err_status_cnt[33];
3352 }
3353
3354 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3355                                 const struct cntr_entry *entry,
3356                                 void *context, int vl, int mode, u64 data)
3357 {
3358         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3359
3360         return dd->send_egress_err_status_cnt[32];
3361 }
3362
3363 static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3364                                 const struct cntr_entry *entry,
3365                                 void *context, int vl, int mode, u64 data)
3366 {
3367         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3368
3369         return dd->send_egress_err_status_cnt[31];
3370 }
3371
3372 static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3373                                 const struct cntr_entry *entry,
3374                                 void *context, int vl, int mode, u64 data)
3375 {
3376         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3377
3378         return dd->send_egress_err_status_cnt[30];
3379 }
3380
3381 static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3382                                 const struct cntr_entry *entry,
3383                                 void *context, int vl, int mode, u64 data)
3384 {
3385         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3386
3387         return dd->send_egress_err_status_cnt[29];
3388 }
3389
3390 static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3391                                 const struct cntr_entry *entry,
3392                                 void *context, int vl, int mode, u64 data)
3393 {
3394         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3395
3396         return dd->send_egress_err_status_cnt[28];
3397 }
3398
3399 static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3400                                 const struct cntr_entry *entry,
3401                                 void *context, int vl, int mode, u64 data)
3402 {
3403         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3404
3405         return dd->send_egress_err_status_cnt[27];
3406 }
3407
3408 static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3409                                 const struct cntr_entry *entry,
3410                                 void *context, int vl, int mode, u64 data)
3411 {
3412         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3413
3414         return dd->send_egress_err_status_cnt[26];
3415 }
3416
3417 static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3418                                 const struct cntr_entry *entry,
3419                                 void *context, int vl, int mode, u64 data)
3420 {
3421         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3422
3423         return dd->send_egress_err_status_cnt[25];
3424 }
3425
3426 static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3427                                 const struct cntr_entry *entry,
3428                                 void *context, int vl, int mode, u64 data)
3429 {
3430         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3431
3432         return dd->send_egress_err_status_cnt[24];
3433 }
3434
3435 static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3436                                 const struct cntr_entry *entry,
3437                                 void *context, int vl, int mode, u64 data)
3438 {
3439         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3440
3441         return dd->send_egress_err_status_cnt[23];
3442 }
3443
3444 static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3445                                 const struct cntr_entry *entry,
3446                                 void *context, int vl, int mode, u64 data)
3447 {
3448         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3449
3450         return dd->send_egress_err_status_cnt[22];
3451 }
3452
3453 static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3454                                 const struct cntr_entry *entry,
3455                                 void *context, int vl, int mode, u64 data)
3456 {
3457         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3458
3459         return dd->send_egress_err_status_cnt[21];
3460 }
3461
3462 static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3463                                 const struct cntr_entry *entry,
3464                                 void *context, int vl, int mode, u64 data)
3465 {
3466         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3467
3468         return dd->send_egress_err_status_cnt[20];
3469 }
3470
3471 static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3472                                 const struct cntr_entry *entry,
3473                                 void *context, int vl, int mode, u64 data)
3474 {
3475         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3476
3477         return dd->send_egress_err_status_cnt[19];
3478 }
3479
3480 static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3481                                 const struct cntr_entry *entry,
3482                                 void *context, int vl, int mode, u64 data)
3483 {
3484         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3485
3486         return dd->send_egress_err_status_cnt[18];
3487 }
3488
3489 static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3490                                 const struct cntr_entry *entry,
3491                                 void *context, int vl, int mode, u64 data)
3492 {
3493         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3494
3495         return dd->send_egress_err_status_cnt[17];
3496 }
3497
3498 static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3499                                 const struct cntr_entry *entry,
3500                                 void *context, int vl, int mode, u64 data)
3501 {
3502         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3503
3504         return dd->send_egress_err_status_cnt[16];
3505 }
3506
3507 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3508                                            void *context, int vl, int mode,
3509                                            u64 data)
3510 {
3511         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3512
3513         return dd->send_egress_err_status_cnt[15];
3514 }
3515
3516 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3517                                                  void *context, int vl,
3518                                                  int mode, u64 data)
3519 {
3520         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3521
3522         return dd->send_egress_err_status_cnt[14];
3523 }
3524
3525 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3526                                                void *context, int vl, int mode,
3527                                                u64 data)
3528 {
3529         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3530
3531         return dd->send_egress_err_status_cnt[13];
3532 }
3533
3534 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3535                                         void *context, int vl, int mode,
3536                                         u64 data)
3537 {
3538         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3539
3540         return dd->send_egress_err_status_cnt[12];
3541 }
3542
3543 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3544                                 const struct cntr_entry *entry,
3545                                 void *context, int vl, int mode, u64 data)
3546 {
3547         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3548
3549         return dd->send_egress_err_status_cnt[11];
3550 }
3551
3552 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3553                                              void *context, int vl, int mode,
3554                                              u64 data)
3555 {
3556         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3557
3558         return dd->send_egress_err_status_cnt[10];
3559 }
3560
3561 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3562                                             void *context, int vl, int mode,
3563                                             u64 data)
3564 {
3565         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3566
3567         return dd->send_egress_err_status_cnt[9];
3568 }
3569
3570 static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3571                                 const struct cntr_entry *entry,
3572                                 void *context, int vl, int mode, u64 data)
3573 {
3574         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3575
3576         return dd->send_egress_err_status_cnt[8];
3577 }
3578
3579 static u64 access_tx_pio_launch_intf_parity_err_cnt(
3580                                 const struct cntr_entry *entry,
3581                                 void *context, int vl, int mode, u64 data)
3582 {
3583         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3584
3585         return dd->send_egress_err_status_cnt[7];
3586 }
3587
3588 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3589                                             void *context, int vl, int mode,
3590                                             u64 data)
3591 {
3592         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3593
3594         return dd->send_egress_err_status_cnt[6];
3595 }
3596
3597 static u64 access_tx_incorrect_link_state_err_cnt(
3598                                 const struct cntr_entry *entry,
3599                                 void *context, int vl, int mode, u64 data)
3600 {
3601         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3602
3603         return dd->send_egress_err_status_cnt[5];
3604 }
3605
3606 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3607                                       void *context, int vl, int mode,
3608                                       u64 data)
3609 {
3610         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3611
3612         return dd->send_egress_err_status_cnt[4];
3613 }
3614
3615 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3616                                 const struct cntr_entry *entry,
3617                                 void *context, int vl, int mode, u64 data)
3618 {
3619         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3620
3621         return dd->send_egress_err_status_cnt[3];
3622 }
3623
3624 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3625                                             void *context, int vl, int mode,
3626                                             u64 data)
3627 {
3628         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3629
3630         return dd->send_egress_err_status_cnt[2];
3631 }
3632
3633 static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3634                                 const struct cntr_entry *entry,
3635                                 void *context, int vl, int mode, u64 data)
3636 {
3637         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3638
3639         return dd->send_egress_err_status_cnt[1];
3640 }
3641
3642 static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3643                                 const struct cntr_entry *entry,
3644                                 void *context, int vl, int mode, u64 data)
3645 {
3646         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3647
3648         return dd->send_egress_err_status_cnt[0];
3649 }
3650
3651 /*
3652  * Software counters corresponding to each of the
3653  * error status bits within SendErrStatus
3654  */
3655 static u64 access_send_csr_write_bad_addr_err_cnt(
3656                                 const struct cntr_entry *entry,
3657                                 void *context, int vl, int mode, u64 data)
3658 {
3659         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3660
3661         return dd->send_err_status_cnt[2];
3662 }
3663
3664 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3665                                                  void *context, int vl,
3666                                                  int mode, u64 data)
3667 {
3668         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3669
3670         return dd->send_err_status_cnt[1];
3671 }
3672
3673 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3674                                       void *context, int vl, int mode,
3675                                       u64 data)
3676 {
3677         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3678
3679         return dd->send_err_status_cnt[0];
3680 }
3681
3682 /*
3683  * Software counters corresponding to each of the
3684  * error status bits within SendCtxtErrStatus
3685  */
3686 static u64 access_pio_write_out_of_bounds_err_cnt(
3687                                 const struct cntr_entry *entry,
3688                                 void *context, int vl, int mode, u64 data)
3689 {
3690         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3691
3692         return dd->sw_ctxt_err_status_cnt[4];
3693 }
3694
3695 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3696                                              void *context, int vl, int mode,
3697                                              u64 data)
3698 {
3699         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3700
3701         return dd->sw_ctxt_err_status_cnt[3];
3702 }
3703
3704 static u64 access_pio_write_crosses_boundary_err_cnt(
3705                                 const struct cntr_entry *entry,
3706                                 void *context, int vl, int mode, u64 data)
3707 {
3708         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3709
3710         return dd->sw_ctxt_err_status_cnt[2];
3711 }
3712
3713 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3714                                                 void *context, int vl,
3715                                                 int mode, u64 data)
3716 {
3717         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3718
3719         return dd->sw_ctxt_err_status_cnt[1];
3720 }
3721
3722 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3723                                                void *context, int vl, int mode,
3724                                                u64 data)
3725 {
3726         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3727
3728         return dd->sw_ctxt_err_status_cnt[0];
3729 }
3730
3731 /*
3732  * Software counters corresponding to each of the
3733  * error status bits within SendDmaEngErrStatus
3734  */
3735 static u64 access_sdma_header_request_fifo_cor_err_cnt(
3736                                 const struct cntr_entry *entry,
3737                                 void *context, int vl, int mode, u64 data)
3738 {
3739         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3740
3741         return dd->sw_send_dma_eng_err_status_cnt[23];
3742 }
3743
3744 static u64 access_sdma_header_storage_cor_err_cnt(
3745                                 const struct cntr_entry *entry,
3746                                 void *context, int vl, int mode, u64 data)
3747 {
3748         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3749
3750         return dd->sw_send_dma_eng_err_status_cnt[22];
3751 }
3752
3753 static u64 access_sdma_packet_tracking_cor_err_cnt(
3754                                 const struct cntr_entry *entry,
3755                                 void *context, int vl, int mode, u64 data)
3756 {
3757         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3758
3759         return dd->sw_send_dma_eng_err_status_cnt[21];
3760 }
3761
3762 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3763                                             void *context, int vl, int mode,
3764                                             u64 data)
3765 {
3766         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3767
3768         return dd->sw_send_dma_eng_err_status_cnt[20];
3769 }
3770
3771 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3772                                               void *context, int vl, int mode,
3773                                               u64 data)
3774 {
3775         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3776
3777         return dd->sw_send_dma_eng_err_status_cnt[19];
3778 }
3779
3780 static u64 access_sdma_header_request_fifo_unc_err_cnt(
3781                                 const struct cntr_entry *entry,
3782                                 void *context, int vl, int mode, u64 data)
3783 {
3784         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3785
3786         return dd->sw_send_dma_eng_err_status_cnt[18];
3787 }
3788
3789 static u64 access_sdma_header_storage_unc_err_cnt(
3790                                 const struct cntr_entry *entry,
3791                                 void *context, int vl, int mode, u64 data)
3792 {
3793         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3794
3795         return dd->sw_send_dma_eng_err_status_cnt[17];
3796 }
3797
3798 static u64 access_sdma_packet_tracking_unc_err_cnt(
3799                                 const struct cntr_entry *entry,
3800                                 void *context, int vl, int mode, u64 data)
3801 {
3802         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3803
3804         return dd->sw_send_dma_eng_err_status_cnt[16];
3805 }
3806
3807 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3808                                             void *context, int vl, int mode,
3809                                             u64 data)
3810 {
3811         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3812
3813         return dd->sw_send_dma_eng_err_status_cnt[15];
3814 }
3815
3816 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3817                                               void *context, int vl, int mode,
3818                                               u64 data)
3819 {
3820         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3821
3822         return dd->sw_send_dma_eng_err_status_cnt[14];
3823 }
3824
3825 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3826                                        void *context, int vl, int mode,
3827                                        u64 data)
3828 {
3829         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3830
3831         return dd->sw_send_dma_eng_err_status_cnt[13];
3832 }
3833
3834 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3835                                              void *context, int vl, int mode,
3836                                              u64 data)
3837 {
3838         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3839
3840         return dd->sw_send_dma_eng_err_status_cnt[12];
3841 }
3842
3843 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3844                                               void *context, int vl, int mode,
3845                                               u64 data)
3846 {
3847         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3848
3849         return dd->sw_send_dma_eng_err_status_cnt[11];
3850 }
3851
3852 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3853                                              void *context, int vl, int mode,
3854                                              u64 data)
3855 {
3856         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3857
3858         return dd->sw_send_dma_eng_err_status_cnt[10];
3859 }
3860
3861 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3862                                           void *context, int vl, int mode,
3863                                           u64 data)
3864 {
3865         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3866
3867         return dd->sw_send_dma_eng_err_status_cnt[9];
3868 }
3869
3870 static u64 access_sdma_packet_desc_overflow_err_cnt(
3871                                 const struct cntr_entry *entry,
3872                                 void *context, int vl, int mode, u64 data)
3873 {
3874         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3875
3876         return dd->sw_send_dma_eng_err_status_cnt[8];
3877 }
3878
3879 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3880                                                void *context, int vl,
3881                                                int mode, u64 data)
3882 {
3883         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3884
3885         return dd->sw_send_dma_eng_err_status_cnt[7];
3886 }
3887
3888 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3889                                     void *context, int vl, int mode, u64 data)
3890 {
3891         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3892
3893         return dd->sw_send_dma_eng_err_status_cnt[6];
3894 }
3895
3896 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3897                                         void *context, int vl, int mode,
3898                                         u64 data)
3899 {
3900         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3901
3902         return dd->sw_send_dma_eng_err_status_cnt[5];
3903 }
3904
3905 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3906                                           void *context, int vl, int mode,
3907                                           u64 data)
3908 {
3909         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3910
3911         return dd->sw_send_dma_eng_err_status_cnt[4];
3912 }
3913
3914 static u64 access_sdma_tail_out_of_bounds_err_cnt(
3915                                 const struct cntr_entry *entry,
3916                                 void *context, int vl, int mode, u64 data)
3917 {
3918         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3919
3920         return dd->sw_send_dma_eng_err_status_cnt[3];
3921 }
3922
3923 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
3924                                         void *context, int vl, int mode,
3925                                         u64 data)
3926 {
3927         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3928
3929         return dd->sw_send_dma_eng_err_status_cnt[2];
3930 }
3931
3932 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
3933                                             void *context, int vl, int mode,
3934                                             u64 data)
3935 {
3936         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3937
3938         return dd->sw_send_dma_eng_err_status_cnt[1];
3939 }
3940
3941 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
3942                                         void *context, int vl, int mode,
3943                                         u64 data)
3944 {
3945         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3946
3947         return dd->sw_send_dma_eng_err_status_cnt[0];
3948 }
3949
3950 #define def_access_sw_cpu(cntr) \
3951 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry,               \
3952                               void *context, int vl, int mode, u64 data)      \
3953 {                                                                             \
3954         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;        \
3955         return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr,       \
3956                               ppd->ibport_data.rvp.cntr, vl,                  \
3957                               mode, data);                                    \
3958 }
3959
3960 def_access_sw_cpu(rc_acks);
3961 def_access_sw_cpu(rc_qacks);
3962 def_access_sw_cpu(rc_delayed_comp);
3963
3964 #define def_access_ibp_counter(cntr) \
3965 static u64 access_ibp_##cntr(const struct cntr_entry *entry,                  \
3966                                 void *context, int vl, int mode, u64 data)    \
3967 {                                                                             \
3968         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;        \
3969                                                                               \
3970         if (vl != CNTR_INVALID_VL)                                            \
3971                 return 0;                                                     \
3972                                                                               \
3973         return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr,        \
3974                              mode, data);                                     \
3975 }
3976
3977 def_access_ibp_counter(loop_pkts);
3978 def_access_ibp_counter(rc_resends);
3979 def_access_ibp_counter(rnr_naks);
3980 def_access_ibp_counter(other_naks);
3981 def_access_ibp_counter(rc_timeouts);
3982 def_access_ibp_counter(pkt_drops);
3983 def_access_ibp_counter(dmawait);
3984 def_access_ibp_counter(rc_seqnak);
3985 def_access_ibp_counter(rc_dupreq);
3986 def_access_ibp_counter(rdma_seq);
3987 def_access_ibp_counter(unaligned);
3988 def_access_ibp_counter(seq_naks);
3989
3990 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
3991 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
3992 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
3993                         CNTR_NORMAL),
3994 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
3995                         CNTR_NORMAL),
3996 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
3997                         RCV_TID_FLOW_GEN_MISMATCH_CNT,
3998                         CNTR_NORMAL),
3999 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4000                         CNTR_NORMAL),
4001 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4002                         RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4003 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4004                         CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4005 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4006                         CNTR_NORMAL),
4007 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4008                         CNTR_NORMAL),
4009 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4010                         CNTR_NORMAL),
4011 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4012                         CNTR_NORMAL),
4013 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4014                         CNTR_NORMAL),
4015 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4016                         CNTR_NORMAL),
4017 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4018                         CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4019 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4020                         CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4021 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4022                               CNTR_SYNTH),
4023 [C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH),
4024 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4025                                  CNTR_SYNTH),
4026 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4027                                   CNTR_SYNTH),
4028 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4029                                   CNTR_SYNTH),
4030 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4031                                    DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4032 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4033                                   DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4034                                   CNTR_SYNTH),
4035 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4036                                 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4037 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4038                                CNTR_SYNTH),
4039 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4040                               CNTR_SYNTH),
4041 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4042                                CNTR_SYNTH),
4043 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4044                                  CNTR_SYNTH),
4045 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4046                                 CNTR_SYNTH),
4047 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4048                                 CNTR_SYNTH),
4049 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4050                                CNTR_SYNTH),
4051 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4052                                  CNTR_SYNTH | CNTR_VL),
4053 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4054                                 CNTR_SYNTH | CNTR_VL),
4055 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4056 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4057                                  CNTR_SYNTH | CNTR_VL),
4058 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4059 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4060                                  CNTR_SYNTH | CNTR_VL),
4061 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4062                               CNTR_SYNTH),
4063 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4064                                  CNTR_SYNTH | CNTR_VL),
4065 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4066                                 CNTR_SYNTH),
4067 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4068                                    CNTR_SYNTH | CNTR_VL),
4069 [C_DC_TOTAL_CRC] =
4070         DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4071                          CNTR_SYNTH),
4072 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4073                                   CNTR_SYNTH),
4074 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4075                                   CNTR_SYNTH),
4076 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4077                                   CNTR_SYNTH),
4078 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4079                                   CNTR_SYNTH),
4080 [C_DC_CRC_MULT_LN] =
4081         DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4082                          CNTR_SYNTH),
4083 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4084                                     CNTR_SYNTH),
4085 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4086                                     CNTR_SYNTH),
4087 [C_DC_SEQ_CRC_CNT] =
4088         DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4089                          CNTR_SYNTH),
4090 [C_DC_ESC0_ONLY_CNT] =
4091         DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4092                          CNTR_SYNTH),
4093 [C_DC_ESC0_PLUS1_CNT] =
4094         DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4095                          CNTR_SYNTH),
4096 [C_DC_ESC0_PLUS2_CNT] =
4097         DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4098                          CNTR_SYNTH),
4099 [C_DC_REINIT_FROM_PEER_CNT] =
4100         DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4101                          CNTR_SYNTH),
4102 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4103                                   CNTR_SYNTH),
4104 [C_DC_MISC_FLG_CNT] =
4105         DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4106                          CNTR_SYNTH),
4107 [C_DC_PRF_GOOD_LTP_CNT] =
4108         DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4109 [C_DC_PRF_ACCEPTED_LTP_CNT] =
4110         DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4111                          CNTR_SYNTH),
4112 [C_DC_PRF_RX_FLIT_CNT] =
4113         DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4114 [C_DC_PRF_TX_FLIT_CNT] =
4115         DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4116 [C_DC_PRF_CLK_CNTR] =
4117         DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4118 [C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4119         DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4120 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4121         DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4122                          CNTR_SYNTH),
4123 [C_DC_PG_STS_TX_SBE_CNT] =
4124         DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4125 [C_DC_PG_STS_TX_MBE_CNT] =
4126         DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4127                          CNTR_SYNTH),
4128 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4129                             access_sw_cpu_intr),
4130 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4131                             access_sw_cpu_rcv_limit),
4132 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4133                             access_sw_vtx_wait),
4134 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4135                             access_sw_pio_wait),
4136 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4137                             access_sw_pio_drain),
4138 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4139                             access_sw_kmem_wait),
4140 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4141                             access_sw_send_schedule),
4142 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4143                                       SEND_DMA_DESC_FETCHED_CNT, 0,
4144                                       CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4145                                       dev_access_u32_csr),
4146 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4147                              CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4148                              access_sde_int_cnt),
4149 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4150                              CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4151                              access_sde_err_cnt),
4152 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4153                                   CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4154                                   access_sde_idle_int_cnt),
4155 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4156                                       CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4157                                       access_sde_progress_int_cnt),
4158 /* MISC_ERR_STATUS */
4159 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4160                                 CNTR_NORMAL,
4161                                 access_misc_pll_lock_fail_err_cnt),
4162 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4163                                 CNTR_NORMAL,
4164                                 access_misc_mbist_fail_err_cnt),
4165 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4166                                 CNTR_NORMAL,
4167                                 access_misc_invalid_eep_cmd_err_cnt),
4168 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4169                                 CNTR_NORMAL,
4170                                 access_misc_efuse_done_parity_err_cnt),
4171 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4172                                 CNTR_NORMAL,
4173                                 access_misc_efuse_write_err_cnt),
4174 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4175                                 0, CNTR_NORMAL,
4176                                 access_misc_efuse_read_bad_addr_err_cnt),
4177 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4178                                 CNTR_NORMAL,
4179                                 access_misc_efuse_csr_parity_err_cnt),
4180 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4181                                 CNTR_NORMAL,
4182                                 access_misc_fw_auth_failed_err_cnt),
4183 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4184                                 CNTR_NORMAL,
4185                                 access_misc_key_mismatch_err_cnt),
4186 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4187                                 CNTR_NORMAL,
4188                                 access_misc_sbus_write_failed_err_cnt),
4189 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4190                                 CNTR_NORMAL,
4191                                 access_misc_csr_write_bad_addr_err_cnt),
4192 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4193                                 CNTR_NORMAL,
4194                                 access_misc_csr_read_bad_addr_err_cnt),
4195 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4196                                 CNTR_NORMAL,
4197                                 access_misc_csr_parity_err_cnt),
4198 /* CceErrStatus */
4199 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4200                                 CNTR_NORMAL,
4201                                 access_sw_cce_err_status_aggregated_cnt),
4202 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4203                                 CNTR_NORMAL,
4204                                 access_cce_msix_csr_parity_err_cnt),
4205 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4206                                 CNTR_NORMAL,
4207                                 access_cce_int_map_unc_err_cnt),
4208 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4209                                 CNTR_NORMAL,
4210                                 access_cce_int_map_cor_err_cnt),
4211 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4212                                 CNTR_NORMAL,
4213                                 access_cce_msix_table_unc_err_cnt),
4214 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4215                                 CNTR_NORMAL,
4216                                 access_cce_msix_table_cor_err_cnt),
4217 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4218                                 0, CNTR_NORMAL,
4219                                 access_cce_rxdma_conv_fifo_parity_err_cnt),
4220 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4221                                 0, CNTR_NORMAL,
4222                                 access_cce_rcpl_async_fifo_parity_err_cnt),
4223 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4224                                 CNTR_NORMAL,
4225                                 access_cce_seg_write_bad_addr_err_cnt),
4226 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4227                                 CNTR_NORMAL,
4228                                 access_cce_seg_read_bad_addr_err_cnt),
4229 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4230                                 CNTR_NORMAL,
4231                                 access_la_triggered_cnt),
4232 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4233                                 CNTR_NORMAL,
4234                                 access_cce_trgt_cpl_timeout_err_cnt),
4235 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4236                                 CNTR_NORMAL,
4237                                 access_pcic_receive_parity_err_cnt),
4238 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4239                                 CNTR_NORMAL,
4240                                 access_pcic_transmit_back_parity_err_cnt),
4241 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4242                                 0, CNTR_NORMAL,
4243                                 access_pcic_transmit_front_parity_err_cnt),
4244 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4245                                 CNTR_NORMAL,
4246                                 access_pcic_cpl_dat_q_unc_err_cnt),
4247 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4248                                 CNTR_NORMAL,
4249                                 access_pcic_cpl_hd_q_unc_err_cnt),
4250 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4251                                 CNTR_NORMAL,
4252                                 access_pcic_post_dat_q_unc_err_cnt),
4253 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4254                                 CNTR_NORMAL,
4255                                 access_pcic_post_hd_q_unc_err_cnt),
4256 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4257                                 CNTR_NORMAL,
4258                                 access_pcic_retry_sot_mem_unc_err_cnt),
4259 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4260                                 CNTR_NORMAL,
4261                                 access_pcic_retry_mem_unc_err),
4262 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4263                                 CNTR_NORMAL,
4264                                 access_pcic_n_post_dat_q_parity_err_cnt),
4265 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4266                                 CNTR_NORMAL,
4267                                 access_pcic_n_post_h_q_parity_err_cnt),
4268 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4269                                 CNTR_NORMAL,
4270                                 access_pcic_cpl_dat_q_cor_err_cnt),
4271 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4272                                 CNTR_NORMAL,
4273                                 access_pcic_cpl_hd_q_cor_err_cnt),
4274 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4275                                 CNTR_NORMAL,
4276                                 access_pcic_post_dat_q_cor_err_cnt),
4277 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4278                                 CNTR_NORMAL,
4279                                 access_pcic_post_hd_q_cor_err_cnt),
4280 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4281                                 CNTR_NORMAL,
4282                                 access_pcic_retry_sot_mem_cor_err_cnt),
4283 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4284                                 CNTR_NORMAL,
4285                                 access_pcic_retry_mem_cor_err_cnt),
4286 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4287                                 "CceCli1AsyncFifoDbgParityError", 0, 0,
4288                                 CNTR_NORMAL,
4289                                 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4290 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4291                                 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4292                                 CNTR_NORMAL,
4293                                 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4294                                 ),
4295 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4296                         "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4297                         CNTR_NORMAL,
4298                         access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4299 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4300                         "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4301                         CNTR_NORMAL,
4302                         access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4303 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4304                         0, CNTR_NORMAL,
4305                         access_cce_cli2_async_fifo_parity_err_cnt),
4306 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4307                         CNTR_NORMAL,
4308                         access_cce_csr_cfg_bus_parity_err_cnt),
4309 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4310                         0, CNTR_NORMAL,
4311                         access_cce_cli0_async_fifo_parity_err_cnt),
4312 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4313                         CNTR_NORMAL,
4314                         access_cce_rspd_data_parity_err_cnt),
4315 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4316                         CNTR_NORMAL,
4317                         access_cce_trgt_access_err_cnt),
4318 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4319                         0, CNTR_NORMAL,
4320                         access_cce_trgt_async_fifo_parity_err_cnt),
4321 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4322                         CNTR_NORMAL,
4323                         access_cce_csr_write_bad_addr_err_cnt),
4324 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4325                         CNTR_NORMAL,
4326                         access_cce_csr_read_bad_addr_err_cnt),
4327 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4328                         CNTR_NORMAL,
4329                         access_ccs_csr_parity_err_cnt),
4330
4331 /* RcvErrStatus */
4332 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4333                         CNTR_NORMAL,
4334                         access_rx_csr_parity_err_cnt),
4335 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4336                         CNTR_NORMAL,
4337                         access_rx_csr_write_bad_addr_err_cnt),
4338 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4339                         CNTR_NORMAL,
4340                         access_rx_csr_read_bad_addr_err_cnt),
4341 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4342                         CNTR_NORMAL,
4343                         access_rx_dma_csr_unc_err_cnt),
4344 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4345                         CNTR_NORMAL,
4346                         access_rx_dma_dq_fsm_encoding_err_cnt),
4347 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4348                         CNTR_NORMAL,
4349                         access_rx_dma_eq_fsm_encoding_err_cnt),
4350 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4351                         CNTR_NORMAL,
4352                         access_rx_dma_csr_parity_err_cnt),
4353 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4354                         CNTR_NORMAL,
4355                         access_rx_rbuf_data_cor_err_cnt),
4356 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4357                         CNTR_NORMAL,
4358                         access_rx_rbuf_data_unc_err_cnt),
4359 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4360                         CNTR_NORMAL,
4361                         access_rx_dma_data_fifo_rd_cor_err_cnt),
4362 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4363                         CNTR_NORMAL,
4364                         access_rx_dma_data_fifo_rd_unc_err_cnt),
4365 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4366                         CNTR_NORMAL,
4367                         access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4368 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4369                         CNTR_NORMAL,
4370                         access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4371 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4372                         CNTR_NORMAL,
4373                         access_rx_rbuf_desc_part2_cor_err_cnt),
4374 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4375                         CNTR_NORMAL,
4376                         access_rx_rbuf_desc_part2_unc_err_cnt),
4377 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4378                         CNTR_NORMAL,
4379                         access_rx_rbuf_desc_part1_cor_err_cnt),
4380 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4381                         CNTR_NORMAL,
4382                         access_rx_rbuf_desc_part1_unc_err_cnt),
4383 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4384                         CNTR_NORMAL,
4385                         access_rx_hq_intr_fsm_err_cnt),
4386 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4387                         CNTR_NORMAL,
4388                         access_rx_hq_intr_csr_parity_err_cnt),
4389 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4390                         CNTR_NORMAL,
4391                         access_rx_lookup_csr_parity_err_cnt),
4392 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4393                         CNTR_NORMAL,
4394                         access_rx_lookup_rcv_array_cor_err_cnt),
4395 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4396                         CNTR_NORMAL,
4397                         access_rx_lookup_rcv_array_unc_err_cnt),
4398 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4399                         0, CNTR_NORMAL,
4400                         access_rx_lookup_des_part2_parity_err_cnt),
4401 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4402                         0, CNTR_NORMAL,
4403                         access_rx_lookup_des_part1_unc_cor_err_cnt),
4404 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4405                         CNTR_NORMAL,
4406                         access_rx_lookup_des_part1_unc_err_cnt),
4407 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4408                         CNTR_NORMAL,
4409                         access_rx_rbuf_next_free_buf_cor_err_cnt),
4410 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4411                         CNTR_NORMAL,
4412                         access_rx_rbuf_next_free_buf_unc_err_cnt),
4413 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4414                         "RxRbufFlInitWrAddrParityErr", 0, 0,
4415                         CNTR_NORMAL,
4416                         access_rbuf_fl_init_wr_addr_parity_err_cnt),
4417 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4418                         0, CNTR_NORMAL,
4419                         access_rx_rbuf_fl_initdone_parity_err_cnt),
4420 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4421                         0, CNTR_NORMAL,
4422                         access_rx_rbuf_fl_write_addr_parity_err_cnt),
4423 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4424                         CNTR_NORMAL,
4425                         access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4426 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4427                         CNTR_NORMAL,
4428                         access_rx_rbuf_empty_err_cnt),
4429 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4430                         CNTR_NORMAL,
4431                         access_rx_rbuf_full_err_cnt),
4432 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4433                         CNTR_NORMAL,
4434                         access_rbuf_bad_lookup_err_cnt),
4435 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4436                         CNTR_NORMAL,
4437                         access_rbuf_ctx_id_parity_err_cnt),
4438 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4439                         CNTR_NORMAL,
4440                         access_rbuf_csr_qeopdw_parity_err_cnt),
4441 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4442                         "RxRbufCsrQNumOfPktParityErr", 0, 0,
4443                         CNTR_NORMAL,
4444                         access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4445 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4446                         "RxRbufCsrQTlPtrParityErr", 0, 0,
4447                         CNTR_NORMAL,
4448                         access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4449 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4450                         0, CNTR_NORMAL,
4451                         access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4452 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4453                         0, CNTR_NORMAL,
4454                         access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4455 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4456                         0, 0, CNTR_NORMAL,
4457                         access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4458 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4459                         0, CNTR_NORMAL,
4460                         access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4461 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4462                         "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4463                         CNTR_NORMAL,
4464                         access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4465 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4466                         0, CNTR_NORMAL,
4467                         access_rx_rbuf_block_list_read_cor_err_cnt),
4468 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4469                         0, CNTR_NORMAL,
4470                         access_rx_rbuf_block_list_read_unc_err_cnt),
4471 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4472                         CNTR_NORMAL,
4473                         access_rx_rbuf_lookup_des_cor_err_cnt),
4474 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4475                         CNTR_NORMAL,
4476                         access_rx_rbuf_lookup_des_unc_err_cnt),
4477 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4478                         "RxRbufLookupDesRegUncCorErr", 0, 0,
4479                         CNTR_NORMAL,
4480                         access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4481 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4482                         CNTR_NORMAL,
4483                         access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4484 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4485                         CNTR_NORMAL,
4486                         access_rx_rbuf_free_list_cor_err_cnt),
4487 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4488                         CNTR_NORMAL,
4489                         access_rx_rbuf_free_list_unc_err_cnt),
4490 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4491                         CNTR_NORMAL,
4492                         access_rx_rcv_fsm_encoding_err_cnt),
4493 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4494                         CNTR_NORMAL,
4495                         access_rx_dma_flag_cor_err_cnt),
4496 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4497                         CNTR_NORMAL,
4498                         access_rx_dma_flag_unc_err_cnt),
4499 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4500                         CNTR_NORMAL,
4501                         access_rx_dc_sop_eop_parity_err_cnt),
4502 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4503                         CNTR_NORMAL,
4504                         access_rx_rcv_csr_parity_err_cnt),
4505 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4506                         CNTR_NORMAL,
4507                         access_rx_rcv_qp_map_table_cor_err_cnt),
4508 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4509                         CNTR_NORMAL,
4510                         access_rx_rcv_qp_map_table_unc_err_cnt),
4511 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4512                         CNTR_NORMAL,
4513                         access_rx_rcv_data_cor_err_cnt),
4514 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4515                         CNTR_NORMAL,
4516                         access_rx_rcv_data_unc_err_cnt),
4517 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4518                         CNTR_NORMAL,
4519                         access_rx_rcv_hdr_cor_err_cnt),
4520 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4521                         CNTR_NORMAL,
4522                         access_rx_rcv_hdr_unc_err_cnt),
4523 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4524                         CNTR_NORMAL,
4525                         access_rx_dc_intf_parity_err_cnt),
4526 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4527                         CNTR_NORMAL,
4528                         access_rx_dma_csr_cor_err_cnt),
4529 /* SendPioErrStatus */
4530 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4531                         CNTR_NORMAL,
4532                         access_pio_pec_sop_head_parity_err_cnt),
4533 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4534                         CNTR_NORMAL,
4535                         access_pio_pcc_sop_head_parity_err_cnt),
4536 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4537                         0, 0, CNTR_NORMAL,
4538                         access_pio_last_returned_cnt_parity_err_cnt),
4539 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4540                         0, CNTR_NORMAL,
4541                         access_pio_current_free_cnt_parity_err_cnt),
4542 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4543                         CNTR_NORMAL,
4544                         access_pio_reserved_31_err_cnt),
4545 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4546                         CNTR_NORMAL,
4547                         access_pio_reserved_30_err_cnt),
4548 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4549                         CNTR_NORMAL,
4550                         access_pio_ppmc_sop_len_err_cnt),
4551 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4552                         CNTR_NORMAL,
4553                         access_pio_ppmc_bqc_mem_parity_err_cnt),
4554 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4555                         CNTR_NORMAL,
4556                         access_pio_vl_fifo_parity_err_cnt),
4557 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4558                         CNTR_NORMAL,
4559                         access_pio_vlf_sop_parity_err_cnt),
4560 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4561                         CNTR_NORMAL,
4562                         access_pio_vlf_v1_len_parity_err_cnt),
4563 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4564                         CNTR_NORMAL,
4565                         access_pio_block_qw_count_parity_err_cnt),
4566 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4567                         CNTR_NORMAL,
4568                         access_pio_write_qw_valid_parity_err_cnt),
4569 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4570                         CNTR_NORMAL,
4571                         access_pio_state_machine_err_cnt),
4572 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4573                         CNTR_NORMAL,
4574                         access_pio_write_data_parity_err_cnt),
4575 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4576                         CNTR_NORMAL,
4577                         access_pio_host_addr_mem_cor_err_cnt),
4578 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4579                         CNTR_NORMAL,
4580                         access_pio_host_addr_mem_unc_err_cnt),
4581 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4582                         CNTR_NORMAL,
4583                         access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4584 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4585                         CNTR_NORMAL,
4586                         access_pio_init_sm_in_err_cnt),
4587 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4588                         CNTR_NORMAL,
4589                         access_pio_ppmc_pbl_fifo_err_cnt),
4590 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4591                         0, CNTR_NORMAL,
4592                         access_pio_credit_ret_fifo_parity_err_cnt),
4593 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4594                         CNTR_NORMAL,
4595                         access_pio_v1_len_mem_bank1_cor_err_cnt),
4596 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4597                         CNTR_NORMAL,
4598                         access_pio_v1_len_mem_bank0_cor_err_cnt),
4599 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4600                         CNTR_NORMAL,
4601                         access_pio_v1_len_mem_bank1_unc_err_cnt),
4602 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4603                         CNTR_NORMAL,
4604                         access_pio_v1_len_mem_bank0_unc_err_cnt),
4605 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4606                         CNTR_NORMAL,
4607                         access_pio_sm_pkt_reset_parity_err_cnt),
4608 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4609                         CNTR_NORMAL,
4610                         access_pio_pkt_evict_fifo_parity_err_cnt),
4611 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4612                         "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4613                         CNTR_NORMAL,
4614                         access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4615 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4616                         CNTR_NORMAL,
4617                         access_pio_sbrdctl_crrel_parity_err_cnt),
4618 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4619                         CNTR_NORMAL,
4620                         access_pio_pec_fifo_parity_err_cnt),
4621 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4622                         CNTR_NORMAL,
4623                         access_pio_pcc_fifo_parity_err_cnt),
4624 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4625                         CNTR_NORMAL,
4626                         access_pio_sb_mem_fifo1_err_cnt),
4627 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4628                         CNTR_NORMAL,
4629                         access_pio_sb_mem_fifo0_err_cnt),
4630 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4631                         CNTR_NORMAL,
4632                         access_pio_csr_parity_err_cnt),
4633 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4634                         CNTR_NORMAL,
4635                         access_pio_write_addr_parity_err_cnt),
4636 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4637                         CNTR_NORMAL,
4638                         access_pio_write_bad_ctxt_err_cnt),
4639 /* SendDmaErrStatus */
4640 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4641                         0, CNTR_NORMAL,
4642                         access_sdma_pcie_req_tracking_cor_err_cnt),
4643 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4644                         0, CNTR_NORMAL,
4645                         access_sdma_pcie_req_tracking_unc_err_cnt),
4646 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4647                         CNTR_NORMAL,
4648                         access_sdma_csr_parity_err_cnt),
4649 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4650                         CNTR_NORMAL,
4651                         access_sdma_rpy_tag_err_cnt),
4652 /* SendEgressErrStatus */
4653 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4654                         CNTR_NORMAL,
4655                         access_tx_read_pio_memory_csr_unc_err_cnt),
4656 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4657                         0, CNTR_NORMAL,
4658                         access_tx_read_sdma_memory_csr_err_cnt),
4659 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4660                         CNTR_NORMAL,
4661                         access_tx_egress_fifo_cor_err_cnt),
4662 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4663                         CNTR_NORMAL,
4664                         access_tx_read_pio_memory_cor_err_cnt),
4665 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4666                         CNTR_NORMAL,
4667                         access_tx_read_sdma_memory_cor_err_cnt),
4668 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4669                         CNTR_NORMAL,
4670                         access_tx_sb_hdr_cor_err_cnt),
4671 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4672                         CNTR_NORMAL,
4673                         access_tx_credit_overrun_err_cnt),
4674 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4675                         CNTR_NORMAL,
4676                         access_tx_launch_fifo8_cor_err_cnt),
4677 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4678                         CNTR_NORMAL,
4679                         access_tx_launch_fifo7_cor_err_cnt),
4680 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4681                         CNTR_NORMAL,
4682                         access_tx_launch_fifo6_cor_err_cnt),
4683 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4684                         CNTR_NORMAL,
4685                         access_tx_launch_fifo5_cor_err_cnt),
4686 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4687                         CNTR_NORMAL,
4688                         access_tx_launch_fifo4_cor_err_cnt),
4689 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4690                         CNTR_NORMAL,
4691                         access_tx_launch_fifo3_cor_err_cnt),
4692 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4693                         CNTR_NORMAL,
4694                         access_tx_launch_fifo2_cor_err_cnt),
4695 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4696                         CNTR_NORMAL,
4697                         access_tx_launch_fifo1_cor_err_cnt),
4698 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4699                         CNTR_NORMAL,
4700                         access_tx_launch_fifo0_cor_err_cnt),
4701 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4702                         CNTR_NORMAL,
4703                         access_tx_credit_return_vl_err_cnt),
4704 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4705                         CNTR_NORMAL,
4706                         access_tx_hcrc_insertion_err_cnt),
4707 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4708                         CNTR_NORMAL,
4709                         access_tx_egress_fifo_unc_err_cnt),
4710 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4711                         CNTR_NORMAL,
4712                         access_tx_read_pio_memory_unc_err_cnt),
4713 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4714                         CNTR_NORMAL,
4715                         access_tx_read_sdma_memory_unc_err_cnt),
4716 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4717                         CNTR_NORMAL,
4718                         access_tx_sb_hdr_unc_err_cnt),
4719 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4720                         CNTR_NORMAL,
4721                         access_tx_credit_return_partiy_err_cnt),
4722 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4723                         0, 0, CNTR_NORMAL,
4724                         access_tx_launch_fifo8_unc_or_parity_err_cnt),
4725 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4726                         0, 0, CNTR_NORMAL,
4727                         access_tx_launch_fifo7_unc_or_parity_err_cnt),
4728 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4729                         0, 0, CNTR_NORMAL,
4730                         access_tx_launch_fifo6_unc_or_parity_err_cnt),
4731 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4732                         0, 0, CNTR_NORMAL,
4733                         access_tx_launch_fifo5_unc_or_parity_err_cnt),
4734 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4735                         0, 0, CNTR_NORMAL,
4736                         access_tx_launch_fifo4_unc_or_parity_err_cnt),
4737 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4738                         0, 0, CNTR_NORMAL,
4739                         access_tx_launch_fifo3_unc_or_parity_err_cnt),
4740 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4741                         0, 0, CNTR_NORMAL,
4742                         access_tx_launch_fifo2_unc_or_parity_err_cnt),
4743 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4744                         0, 0, CNTR_NORMAL,
4745                         access_tx_launch_fifo1_unc_or_parity_err_cnt),
4746 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4747                         0, 0, CNTR_NORMAL,
4748                         access_tx_launch_fifo0_unc_or_parity_err_cnt),
4749 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4750                         0, 0, CNTR_NORMAL,
4751                         access_tx_sdma15_disallowed_packet_err_cnt),
4752 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4753                         0, 0, CNTR_NORMAL,
4754                         access_tx_sdma14_disallowed_packet_err_cnt),
4755 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4756                         0, 0, CNTR_NORMAL,
4757                         access_tx_sdma13_disallowed_packet_err_cnt),
4758 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4759                         0, 0, CNTR_NORMAL,
4760                         access_tx_sdma12_disallowed_packet_err_cnt),
4761 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4762                         0, 0, CNTR_NORMAL,
4763                         access_tx_sdma11_disallowed_packet_err_cnt),
4764 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4765                         0, 0, CNTR_NORMAL,
4766                         access_tx_sdma10_disallowed_packet_err_cnt),
4767 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4768                         0, 0, CNTR_NORMAL,
4769                         access_tx_sdma9_disallowed_packet_err_cnt),
4770 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4771                         0, 0, CNTR_NORMAL,
4772                         access_tx_sdma8_disallowed_packet_err_cnt),
4773 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4774                         0, 0, CNTR_NORMAL,
4775                         access_tx_sdma7_disallowed_packet_err_cnt),
4776 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4777                         0, 0, CNTR_NORMAL,
4778                         access_tx_sdma6_disallowed_packet_err_cnt),
4779 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4780                         0, 0, CNTR_NORMAL,
4781                         access_tx_sdma5_disallowed_packet_err_cnt),
4782 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4783                         0, 0, CNTR_NORMAL,
4784                         access_tx_sdma4_disallowed_packet_err_cnt),
4785 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4786                         0, 0, CNTR_NORMAL,
4787                         access_tx_sdma3_disallowed_packet_err_cnt),
4788 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4789                         0, 0, CNTR_NORMAL,
4790                         access_tx_sdma2_disallowed_packet_err_cnt),
4791 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4792                         0, 0, CNTR_NORMAL,
4793                         access_tx_sdma1_disallowed_packet_err_cnt),
4794 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4795                         0, 0, CNTR_NORMAL,
4796                         access_tx_sdma0_disallowed_packet_err_cnt),
4797 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4798                         CNTR_NORMAL,
4799                         access_tx_config_parity_err_cnt),
4800 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4801                         CNTR_NORMAL,
4802                         access_tx_sbrd_ctl_csr_parity_err_cnt),
4803 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4804                         CNTR_NORMAL,
4805                         access_tx_launch_csr_parity_err_cnt),
4806 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4807                         CNTR_NORMAL,
4808                         access_tx_illegal_vl_err_cnt),
4809 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4810                         "TxSbrdCtlStateMachineParityErr", 0, 0,
4811                         CNTR_NORMAL,
4812                         access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4813 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4814                         CNTR_NORMAL,
4815                         access_egress_reserved_10_err_cnt),
4816 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4817                         CNTR_NORMAL,
4818                         access_egress_reserved_9_err_cnt),
4819 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4820                         0, 0, CNTR_NORMAL,
4821                         access_tx_sdma_launch_intf_parity_err_cnt),
4822 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4823                         CNTR_NORMAL,
4824                         access_tx_pio_launch_intf_parity_err_cnt),
4825 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4826                         CNTR_NORMAL,
4827                         access_egress_reserved_6_err_cnt),
4828 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4829                         CNTR_NORMAL,
4830                         access_tx_incorrect_link_state_err_cnt),
4831 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4832                         CNTR_NORMAL,
4833                         access_tx_linkdown_err_cnt),
4834 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4835                         "EgressFifoUnderrunOrParityErr", 0, 0,
4836                         CNTR_NORMAL,
4837                         access_tx_egress_fifi_underrun_or_parity_err_cnt),
4838 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4839                         CNTR_NORMAL,
4840                         access_egress_reserved_2_err_cnt),
4841 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4842                         CNTR_NORMAL,
4843                         access_tx_pkt_integrity_mem_unc_err_cnt),
4844 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4845                         CNTR_NORMAL,
4846                         access_tx_pkt_integrity_mem_cor_err_cnt),
4847 /* SendErrStatus */
4848 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4849                         CNTR_NORMAL,
4850                         access_send_csr_write_bad_addr_err_cnt),
4851 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4852                         CNTR_NORMAL,
4853                         access_send_csr_read_bad_addr_err_cnt),
4854 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4855                         CNTR_NORMAL,
4856                         access_send_csr_parity_cnt),
4857 /* SendCtxtErrStatus */
4858 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4859                         CNTR_NORMAL,
4860                         access_pio_write_out_of_bounds_err_cnt),
4861 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4862                         CNTR_NORMAL,
4863                         access_pio_write_overflow_err_cnt),
4864 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4865                         0, 0, CNTR_NORMAL,
4866                         access_pio_write_crosses_boundary_err_cnt),
4867 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4868                         CNTR_NORMAL,
4869                         access_pio_disallowed_packet_err_cnt),
4870 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4871                         CNTR_NORMAL,
4872                         access_pio_inconsistent_sop_err_cnt),
4873 /* SendDmaEngErrStatus */
4874 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4875                         0, 0, CNTR_NORMAL,
4876                         access_sdma_header_request_fifo_cor_err_cnt),
4877 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4878                         CNTR_NORMAL,
4879                         access_sdma_header_storage_cor_err_cnt),
4880 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4881                         CNTR_NORMAL,
4882                         access_sdma_packet_tracking_cor_err_cnt),
4883 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4884                         CNTR_NORMAL,
4885                         access_sdma_assembly_cor_err_cnt),
4886 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
4887                         CNTR_NORMAL,
4888                         access_sdma_desc_table_cor_err_cnt),
4889 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
4890                         0, 0, CNTR_NORMAL,
4891                         access_sdma_header_request_fifo_unc_err_cnt),
4892 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
4893                         CNTR_NORMAL,
4894                         access_sdma_header_storage_unc_err_cnt),
4895 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
4896                         CNTR_NORMAL,
4897                         access_sdma_packet_tracking_unc_err_cnt),
4898 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
4899                         CNTR_NORMAL,
4900                         access_sdma_assembly_unc_err_cnt),
4901 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
4902                         CNTR_NORMAL,
4903                         access_sdma_desc_table_unc_err_cnt),
4904 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
4905                         CNTR_NORMAL,
4906                         access_sdma_timeout_err_cnt),
4907 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
4908                         CNTR_NORMAL,
4909                         access_sdma_header_length_err_cnt),
4910 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
4911                         CNTR_NORMAL,
4912                         access_sdma_header_address_err_cnt),
4913 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
4914                         CNTR_NORMAL,
4915                         access_sdma_header_select_err_cnt),
4916 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
4917                         CNTR_NORMAL,
4918                         access_sdma_reserved_9_err_cnt),
4919 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
4920                         CNTR_NORMAL,
4921                         access_sdma_packet_desc_overflow_err_cnt),
4922 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
4923                         CNTR_NORMAL,
4924                         access_sdma_length_mismatch_err_cnt),
4925 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
4926                         CNTR_NORMAL,
4927                         access_sdma_halt_err_cnt),
4928 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
4929                         CNTR_NORMAL,
4930                         access_sdma_mem_read_err_cnt),
4931 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
4932                         CNTR_NORMAL,
4933                         access_sdma_first_desc_err_cnt),
4934 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
4935                         CNTR_NORMAL,
4936                         access_sdma_tail_out_of_bounds_err_cnt),
4937 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
4938                         CNTR_NORMAL,
4939                         access_sdma_too_long_err_cnt),
4940 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
4941                         CNTR_NORMAL,
4942                         access_sdma_gen_mismatch_err_cnt),
4943 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
4944                         CNTR_NORMAL,
4945                         access_sdma_wrong_dw_err_cnt),
4946 };
4947
4948 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
4949 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
4950                         CNTR_NORMAL),
4951 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
4952                         CNTR_NORMAL),
4953 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
4954                         CNTR_NORMAL),
4955 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
4956                         CNTR_NORMAL),
4957 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
4958                         CNTR_NORMAL),
4959 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
4960                         CNTR_NORMAL),
4961 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
4962                         CNTR_NORMAL),
4963 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
4964 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
4965 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
4966 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
4967                                       CNTR_SYNTH | CNTR_VL),
4968 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
4969                                      CNTR_SYNTH | CNTR_VL),
4970 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
4971                                       CNTR_SYNTH | CNTR_VL),
4972 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
4973 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
4974 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
4975                              access_sw_link_dn_cnt),
4976 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
4977                            access_sw_link_up_cnt),
4978 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
4979                                  access_sw_unknown_frame_cnt),
4980 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
4981                              access_sw_xmit_discards),
4982 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
4983                                 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
4984                                 access_sw_xmit_discards),
4985 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
4986                                  access_xmit_constraint_errs),
4987 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
4988                                 access_rcv_constraint_errs),
4989 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
4990 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
4991 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
4992 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
4993 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
4994 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
4995 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
4996 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
4997 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
4998 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
4999 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5000 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5001 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5002                                access_sw_cpu_rc_acks),
5003 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
5004                                 access_sw_cpu_rc_qacks),
5005 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
5006                                        access_sw_cpu_rc_delayed_comp),
5007 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5008 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5009 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5010 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5011 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5012 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5013 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5014 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5015 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5016 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5017 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5018 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5019 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5020 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5021 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5022 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5023 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5024 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5025 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5026 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5027 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5028 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5029 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5030 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5031 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5032 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5033 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5034 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5035 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5036 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5037 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5038 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5039 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5040 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5041 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5042 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5043 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5044 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5045 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5046 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5047 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5048 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5049 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5050 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5051 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5052 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5053 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5054 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5055 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5056 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5057 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5058 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5059 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5060 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5061 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5062 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5063 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5064 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5065 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5066 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5067 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5068 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5069 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5070 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5071 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5072 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5073 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5074 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5075 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5076 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5077 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5078 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5079 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5080 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5081 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5082 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5083 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5084 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5085 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5086 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5087 };
5088
5089 /* ======================================================================== */
5090
5091 /* return true if this is chip revision revision a */
5092 int is_ax(struct hfi1_devdata *dd)
5093 {
5094         u8 chip_rev_minor =
5095                 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5096                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
5097         return (chip_rev_minor & 0xf0) == 0;
5098 }
5099
5100 /* return true if this is chip revision revision b */
5101 int is_bx(struct hfi1_devdata *dd)
5102 {
5103         u8 chip_rev_minor =
5104                 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5105                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
5106         return (chip_rev_minor & 0xF0) == 0x10;
5107 }
5108
5109 /*
5110  * Append string s to buffer buf.  Arguments curp and len are the current
5111  * position and remaining length, respectively.
5112  *
5113  * return 0 on success, 1 on out of room
5114  */
5115 static int append_str(char *buf, char **curp, int *lenp, const char *s)
5116 {
5117         char *p = *curp;
5118         int len = *lenp;
5119         int result = 0; /* success */
5120         char c;
5121
5122         /* add a comma, if first in the buffer */
5123         if (p != buf) {
5124                 if (len == 0) {
5125                         result = 1; /* out of room */
5126                         goto done;
5127                 }
5128                 *p++ = ',';
5129                 len--;
5130         }
5131
5132         /* copy the string */
5133         while ((c = *s++) != 0) {
5134                 if (len == 0) {
5135                         result = 1; /* out of room */
5136                         goto done;
5137                 }
5138                 *p++ = c;
5139                 len--;
5140         }
5141
5142 done:
5143         /* write return values */
5144         *curp = p;
5145         *lenp = len;
5146
5147         return result;
5148 }
5149
5150 /*
5151  * Using the given flag table, print a comma separated string into
5152  * the buffer.  End in '*' if the buffer is too short.
5153  */
5154 static char *flag_string(char *buf, int buf_len, u64 flags,
5155                          struct flag_table *table, int table_size)
5156 {
5157         char extra[32];
5158         char *p = buf;
5159         int len = buf_len;
5160         int no_room = 0;
5161         int i;
5162
5163         /* make sure there is at least 2 so we can form "*" */
5164         if (len < 2)
5165                 return "";
5166
5167         len--;  /* leave room for a nul */
5168         for (i = 0; i < table_size; i++) {
5169                 if (flags & table[i].flag) {
5170                         no_room = append_str(buf, &p, &len, table[i].str);
5171                         if (no_room)
5172                                 break;
5173                         flags &= ~table[i].flag;
5174                 }
5175         }
5176
5177         /* any undocumented bits left? */
5178         if (!no_room && flags) {
5179                 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5180                 no_room = append_str(buf, &p, &len, extra);
5181         }
5182
5183         /* add * if ran out of room */
5184         if (no_room) {
5185                 /* may need to back up to add space for a '*' */
5186                 if (len == 0)
5187                         --p;
5188                 *p++ = '*';
5189         }
5190
5191         /* add final nul - space already allocated above */
5192         *p = 0;
5193         return buf;
5194 }
5195
5196 /* first 8 CCE error interrupt source names */
5197 static const char * const cce_misc_names[] = {
5198         "CceErrInt",            /* 0 */
5199         "RxeErrInt",            /* 1 */
5200         "MiscErrInt",           /* 2 */
5201         "Reserved3",            /* 3 */
5202         "PioErrInt",            /* 4 */
5203         "SDmaErrInt",           /* 5 */
5204         "EgressErrInt",         /* 6 */
5205         "TxeErrInt"             /* 7 */
5206 };
5207
5208 /*
5209  * Return the miscellaneous error interrupt name.
5210  */
5211 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5212 {
5213         if (source < ARRAY_SIZE(cce_misc_names))
5214                 strncpy(buf, cce_misc_names[source], bsize);
5215         else
5216                 snprintf(buf, bsize, "Reserved%u",
5217                          source + IS_GENERAL_ERR_START);
5218
5219         return buf;
5220 }
5221
5222 /*
5223  * Return the SDMA engine error interrupt name.
5224  */
5225 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5226 {
5227         snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5228         return buf;
5229 }
5230
5231 /*
5232  * Return the send context error interrupt name.
5233  */
5234 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5235 {
5236         snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5237         return buf;
5238 }
5239
5240 static const char * const various_names[] = {
5241         "PbcInt",
5242         "GpioAssertInt",
5243         "Qsfp1Int",
5244         "Qsfp2Int",
5245         "TCritInt"
5246 };
5247
5248 /*
5249  * Return the various interrupt name.
5250  */
5251 static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5252 {
5253         if (source < ARRAY_SIZE(various_names))
5254                 strncpy(buf, various_names[source], bsize);
5255         else
5256                 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
5257         return buf;
5258 }
5259
5260 /*
5261  * Return the DC interrupt name.
5262  */
5263 static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5264 {
5265         static const char * const dc_int_names[] = {
5266                 "common",
5267                 "lcb",
5268                 "8051",
5269                 "lbm"   /* local block merge */
5270         };
5271
5272         if (source < ARRAY_SIZE(dc_int_names))
5273                 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5274         else
5275                 snprintf(buf, bsize, "DCInt%u", source);
5276         return buf;
5277 }
5278
5279 static const char * const sdma_int_names[] = {
5280         "SDmaInt",
5281         "SdmaIdleInt",
5282         "SdmaProgressInt",
5283 };
5284
5285 /*
5286  * Return the SDMA engine interrupt name.
5287  */
5288 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5289 {
5290         /* what interrupt */
5291         unsigned int what  = source / TXE_NUM_SDMA_ENGINES;
5292         /* which engine */
5293         unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5294
5295         if (likely(what < 3))
5296                 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5297         else
5298                 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5299         return buf;
5300 }
5301
5302 /*
5303  * Return the receive available interrupt name.
5304  */
5305 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5306 {
5307         snprintf(buf, bsize, "RcvAvailInt%u", source);
5308         return buf;
5309 }
5310
5311 /*
5312  * Return the receive urgent interrupt name.
5313  */
5314 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5315 {
5316         snprintf(buf, bsize, "RcvUrgentInt%u", source);
5317         return buf;
5318 }
5319
5320 /*
5321  * Return the send credit interrupt name.
5322  */
5323 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5324 {
5325         snprintf(buf, bsize, "SendCreditInt%u", source);
5326         return buf;
5327 }
5328
5329 /*
5330  * Return the reserved interrupt name.
5331  */
5332 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5333 {
5334         snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5335         return buf;
5336 }
5337
5338 static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5339 {
5340         return flag_string(buf, buf_len, flags,
5341                            cce_err_status_flags,
5342                            ARRAY_SIZE(cce_err_status_flags));
5343 }
5344
5345 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5346 {
5347         return flag_string(buf, buf_len, flags,
5348                            rxe_err_status_flags,
5349                            ARRAY_SIZE(rxe_err_status_flags));
5350 }
5351
5352 static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5353 {
5354         return flag_string(buf, buf_len, flags, misc_err_status_flags,
5355                            ARRAY_SIZE(misc_err_status_flags));
5356 }
5357
5358 static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5359 {
5360         return flag_string(buf, buf_len, flags,
5361                            pio_err_status_flags,
5362                            ARRAY_SIZE(pio_err_status_flags));
5363 }
5364
5365 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5366 {
5367         return flag_string(buf, buf_len, flags,
5368                            sdma_err_status_flags,
5369                            ARRAY_SIZE(sdma_err_status_flags));
5370 }
5371
5372 static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5373 {
5374         return flag_string(buf, buf_len, flags,
5375                            egress_err_status_flags,
5376                            ARRAY_SIZE(egress_err_status_flags));
5377 }
5378
5379 static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5380 {
5381         return flag_string(buf, buf_len, flags,
5382                            egress_err_info_flags,
5383                            ARRAY_SIZE(egress_err_info_flags));
5384 }
5385
5386 static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5387 {
5388         return flag_string(buf, buf_len, flags,
5389                            send_err_status_flags,
5390                            ARRAY_SIZE(send_err_status_flags));
5391 }
5392
5393 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5394 {
5395         char buf[96];
5396         int i = 0;
5397
5398         /*
5399          * For most these errors, there is nothing that can be done except
5400          * report or record it.
5401          */
5402         dd_dev_info(dd, "CCE Error: %s\n",
5403                     cce_err_status_string(buf, sizeof(buf), reg));
5404
5405         if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5406             is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
5407                 /* this error requires a manual drop into SPC freeze mode */
5408                 /* then a fix up */
5409                 start_freeze_handling(dd->pport, FREEZE_SELF);
5410         }
5411
5412         for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5413                 if (reg & (1ull << i)) {
5414                         incr_cntr64(&dd->cce_err_status_cnt[i]);
5415                         /* maintain a counter over all cce_err_status errors */
5416                         incr_cntr64(&dd->sw_cce_err_status_aggregate);
5417                 }
5418         }
5419 }
5420
5421 /*
5422  * Check counters for receive errors that do not have an interrupt
5423  * associated with them.
5424  */
5425 #define RCVERR_CHECK_TIME 10
5426 static void update_rcverr_timer(unsigned long opaque)
5427 {
5428         struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
5429         struct hfi1_pportdata *ppd = dd->pport;
5430         u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5431
5432         if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
5433             ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
5434                 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
5435                 set_link_down_reason(
5436                 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5437                 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
5438                 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
5439         }
5440         dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
5441
5442         mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5443 }
5444
5445 static int init_rcverr(struct hfi1_devdata *dd)
5446 {
5447         setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd);
5448         /* Assume the hardware counter has been reset */
5449         dd->rcv_ovfl_cnt = 0;
5450         return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5451 }
5452
5453 static void free_rcverr(struct hfi1_devdata *dd)
5454 {
5455         if (dd->rcverr_timer.data)
5456                 del_timer_sync(&dd->rcverr_timer);
5457         dd->rcverr_timer.data = 0;
5458 }
5459
5460 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5461 {
5462         char buf[96];
5463         int i = 0;
5464
5465         dd_dev_info(dd, "Receive Error: %s\n",
5466                     rxe_err_status_string(buf, sizeof(buf), reg));
5467
5468         if (reg & ALL_RXE_FREEZE_ERR) {
5469                 int flags = 0;
5470
5471                 /*
5472                  * Freeze mode recovery is disabled for the errors
5473                  * in RXE_FREEZE_ABORT_MASK
5474                  */
5475                 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
5476                         flags = FREEZE_ABORT;
5477
5478                 start_freeze_handling(dd->pport, flags);
5479         }
5480
5481         for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5482                 if (reg & (1ull << i))
5483                         incr_cntr64(&dd->rcv_err_status_cnt[i]);
5484         }
5485 }
5486
5487 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5488 {
5489         char buf[96];
5490         int i = 0;
5491
5492         dd_dev_info(dd, "Misc Error: %s",
5493                     misc_err_status_string(buf, sizeof(buf), reg));
5494         for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5495                 if (reg & (1ull << i))
5496                         incr_cntr64(&dd->misc_err_status_cnt[i]);
5497         }
5498 }
5499
5500 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5501 {
5502         char buf[96];
5503         int i = 0;
5504
5505         dd_dev_info(dd, "PIO Error: %s\n",
5506                     pio_err_status_string(buf, sizeof(buf), reg));
5507
5508         if (reg & ALL_PIO_FREEZE_ERR)
5509                 start_freeze_handling(dd->pport, 0);
5510
5511         for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5512                 if (reg & (1ull << i))
5513                         incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5514         }
5515 }
5516
5517 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5518 {
5519         char buf[96];
5520         int i = 0;
5521
5522         dd_dev_info(dd, "SDMA Error: %s\n",
5523                     sdma_err_status_string(buf, sizeof(buf), reg));
5524
5525         if (reg & ALL_SDMA_FREEZE_ERR)
5526                 start_freeze_handling(dd->pport, 0);
5527
5528         for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5529                 if (reg & (1ull << i))
5530                         incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5531         }
5532 }
5533
5534 static inline void __count_port_discards(struct hfi1_pportdata *ppd)
5535 {
5536         incr_cntr64(&ppd->port_xmit_discards);
5537 }
5538
5539 static void count_port_inactive(struct hfi1_devdata *dd)
5540 {
5541         __count_port_discards(dd->pport);
5542 }
5543
5544 /*
5545  * We have had a "disallowed packet" error during egress. Determine the
5546  * integrity check which failed, and update relevant error counter, etc.
5547  *
5548  * Note that the SEND_EGRESS_ERR_INFO register has only a single
5549  * bit of state per integrity check, and so we can miss the reason for an
5550  * egress error if more than one packet fails the same integrity check
5551  * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5552  */
5553 static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5554                                         int vl)
5555 {
5556         struct hfi1_pportdata *ppd = dd->pport;
5557         u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5558         u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5559         char buf[96];
5560
5561         /* clear down all observed info as quickly as possible after read */
5562         write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5563
5564         dd_dev_info(dd,
5565                     "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5566                     info, egress_err_info_string(buf, sizeof(buf), info), src);
5567
5568         /* Eventually add other counters for each bit */
5569         if (info & PORT_DISCARD_EGRESS_ERRS) {
5570                 int weight, i;
5571
5572                 /*
5573                  * Count all applicable bits as individual errors and
5574                  * attribute them to the packet that triggered this handler.
5575                  * This may not be completely accurate due to limitations
5576                  * on the available hardware error information.  There is
5577                  * a single information register and any number of error
5578                  * packets may have occurred and contributed to it before
5579                  * this routine is called.  This means that:
5580                  * a) If multiple packets with the same error occur before
5581                  *    this routine is called, earlier packets are missed.
5582                  *    There is only a single bit for each error type.
5583                  * b) Errors may not be attributed to the correct VL.
5584                  *    The driver is attributing all bits in the info register
5585                  *    to the packet that triggered this call, but bits
5586                  *    could be an accumulation of different packets with
5587                  *    different VLs.
5588                  * c) A single error packet may have multiple counts attached
5589                  *    to it.  There is no way for the driver to know if
5590                  *    multiple bits set in the info register are due to a
5591                  *    single packet or multiple packets.  The driver assumes
5592                  *    multiple packets.
5593                  */
5594                 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
5595                 for (i = 0; i < weight; i++) {
5596                         __count_port_discards(ppd);
5597                         if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5598                                 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5599                         else if (vl == 15)
5600                                 incr_cntr64(&ppd->port_xmit_discards_vl
5601                                             [C_VL_15]);
5602                 }
5603         }
5604 }
5605
5606 /*
5607  * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5608  * register. Does it represent a 'port inactive' error?
5609  */
5610 static inline int port_inactive_err(u64 posn)
5611 {
5612         return (posn >= SEES(TX_LINKDOWN) &&
5613                 posn <= SEES(TX_INCORRECT_LINK_STATE));
5614 }
5615
5616 /*
5617  * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5618  * register. Does it represent a 'disallowed packet' error?
5619  */
5620 static inline int disallowed_pkt_err(int posn)
5621 {
5622         return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5623                 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5624 }
5625
5626 /*
5627  * Input value is a bit position of one of the SDMA engine disallowed
5628  * packet errors.  Return which engine.  Use of this must be guarded by
5629  * disallowed_pkt_err().
5630  */
5631 static inline int disallowed_pkt_engine(int posn)
5632 {
5633         return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5634 }
5635
5636 /*
5637  * Translate an SDMA engine to a VL.  Return -1 if the tranlation cannot
5638  * be done.
5639  */
5640 static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5641 {
5642         struct sdma_vl_map *m;
5643         int vl;
5644
5645         /* range check */
5646         if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5647                 return -1;
5648
5649         rcu_read_lock();
5650         m = rcu_dereference(dd->sdma_map);
5651         vl = m->engine_to_vl[engine];
5652         rcu_read_unlock();
5653
5654         return vl;
5655 }
5656
5657 /*
5658  * Translate the send context (sofware index) into a VL.  Return -1 if the
5659  * translation cannot be done.
5660  */
5661 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5662 {
5663         struct send_context_info *sci;
5664         struct send_context *sc;
5665         int i;
5666
5667         sci = &dd->send_contexts[sw_index];
5668
5669         /* there is no information for user (PSM) and ack contexts */
5670         if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
5671                 return -1;
5672
5673         sc = sci->sc;
5674         if (!sc)
5675                 return -1;
5676         if (dd->vld[15].sc == sc)
5677                 return 15;
5678         for (i = 0; i < num_vls; i++)
5679                 if (dd->vld[i].sc == sc)
5680                         return i;
5681
5682         return -1;
5683 }
5684
5685 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5686 {
5687         u64 reg_copy = reg, handled = 0;
5688         char buf[96];
5689         int i = 0;
5690
5691         if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5692                 start_freeze_handling(dd->pport, 0);
5693         else if (is_ax(dd) &&
5694                  (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5695                  (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
5696                 start_freeze_handling(dd->pport, 0);
5697
5698         while (reg_copy) {
5699                 int posn = fls64(reg_copy);
5700                 /* fls64() returns a 1-based offset, we want it zero based */
5701                 int shift = posn - 1;
5702                 u64 mask = 1ULL << shift;
5703
5704                 if (port_inactive_err(shift)) {
5705                         count_port_inactive(dd);
5706                         handled |= mask;
5707                 } else if (disallowed_pkt_err(shift)) {
5708                         int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5709
5710                         handle_send_egress_err_info(dd, vl);
5711                         handled |= mask;
5712                 }
5713                 reg_copy &= ~mask;
5714         }
5715
5716         reg &= ~handled;
5717
5718         if (reg)
5719                 dd_dev_info(dd, "Egress Error: %s\n",
5720                             egress_err_status_string(buf, sizeof(buf), reg));
5721
5722         for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5723                 if (reg & (1ull << i))
5724                         incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5725         }
5726 }
5727
5728 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5729 {
5730         char buf[96];
5731         int i = 0;
5732
5733         dd_dev_info(dd, "Send Error: %s\n",
5734                     send_err_status_string(buf, sizeof(buf), reg));
5735
5736         for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5737                 if (reg & (1ull << i))
5738                         incr_cntr64(&dd->send_err_status_cnt[i]);
5739         }
5740 }
5741
5742 /*
5743  * The maximum number of times the error clear down will loop before
5744  * blocking a repeating error.  This value is arbitrary.
5745  */
5746 #define MAX_CLEAR_COUNT 20
5747
5748 /*
5749  * Clear and handle an error register.  All error interrupts are funneled
5750  * through here to have a central location to correctly handle single-
5751  * or multi-shot errors.
5752  *
5753  * For non per-context registers, call this routine with a context value
5754  * of 0 so the per-context offset is zero.
5755  *
5756  * If the handler loops too many times, assume that something is wrong
5757  * and can't be fixed, so mask the error bits.
5758  */
5759 static void interrupt_clear_down(struct hfi1_devdata *dd,
5760                                  u32 context,
5761                                  const struct err_reg_info *eri)
5762 {
5763         u64 reg;
5764         u32 count;
5765
5766         /* read in a loop until no more errors are seen */
5767         count = 0;
5768         while (1) {
5769                 reg = read_kctxt_csr(dd, context, eri->status);
5770                 if (reg == 0)
5771                         break;
5772                 write_kctxt_csr(dd, context, eri->clear, reg);
5773                 if (likely(eri->handler))
5774                         eri->handler(dd, context, reg);
5775                 count++;
5776                 if (count > MAX_CLEAR_COUNT) {
5777                         u64 mask;
5778
5779                         dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
5780                                    eri->desc, reg);
5781                         /*
5782                          * Read-modify-write so any other masked bits
5783                          * remain masked.
5784                          */
5785                         mask = read_kctxt_csr(dd, context, eri->mask);
5786                         mask &= ~reg;
5787                         write_kctxt_csr(dd, context, eri->mask, mask);
5788                         break;
5789                 }
5790         }
5791 }
5792
5793 /*
5794  * CCE block "misc" interrupt.  Source is < 16.
5795  */
5796 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5797 {
5798         const struct err_reg_info *eri = &misc_errs[source];
5799
5800         if (eri->handler) {
5801                 interrupt_clear_down(dd, 0, eri);
5802         } else {
5803                 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
5804                            source);
5805         }
5806 }
5807
5808 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5809 {
5810         return flag_string(buf, buf_len, flags,
5811                            sc_err_status_flags,
5812                            ARRAY_SIZE(sc_err_status_flags));
5813 }
5814
5815 /*
5816  * Send context error interrupt.  Source (hw_context) is < 160.
5817  *
5818  * All send context errors cause the send context to halt.  The normal
5819  * clear-down mechanism cannot be used because we cannot clear the
5820  * error bits until several other long-running items are done first.
5821  * This is OK because with the context halted, nothing else is going
5822  * to happen on it anyway.
5823  */
5824 static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5825                                 unsigned int hw_context)
5826 {
5827         struct send_context_info *sci;
5828         struct send_context *sc;
5829         char flags[96];
5830         u64 status;
5831         u32 sw_index;
5832         int i = 0;
5833
5834         sw_index = dd->hw_to_sw[hw_context];
5835         if (sw_index >= dd->num_send_contexts) {
5836                 dd_dev_err(dd,
5837                            "out of range sw index %u for send context %u\n",
5838                            sw_index, hw_context);
5839                 return;
5840         }
5841         sci = &dd->send_contexts[sw_index];
5842         sc = sci->sc;
5843         if (!sc) {
5844                 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
5845                            sw_index, hw_context);
5846                 return;
5847         }
5848
5849         /* tell the software that a halt has begun */
5850         sc_stop(sc, SCF_HALTED);
5851
5852         status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5853
5854         dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
5855                     send_context_err_status_string(flags, sizeof(flags),
5856                                                    status));
5857
5858         if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
5859                 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
5860
5861         /*
5862          * Automatically restart halted kernel contexts out of interrupt
5863          * context.  User contexts must ask the driver to restart the context.
5864          */
5865         if (sc->type != SC_USER)
5866                 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
5867
5868         /*
5869          * Update the counters for the corresponding status bits.
5870          * Note that these particular counters are aggregated over all
5871          * 160 contexts.
5872          */
5873         for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5874                 if (status & (1ull << i))
5875                         incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
5876         }
5877 }
5878
5879 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
5880                                 unsigned int source, u64 status)
5881 {
5882         struct sdma_engine *sde;
5883         int i = 0;
5884
5885         sde = &dd->per_sdma[source];
5886 #ifdef CONFIG_SDMA_VERBOSITY
5887         dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5888                    slashstrip(__FILE__), __LINE__, __func__);
5889         dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
5890                    sde->this_idx, source, (unsigned long long)status);
5891 #endif
5892         sde->err_cnt++;
5893         sdma_engine_error(sde, status);
5894
5895         /*
5896         * Update the counters for the corresponding status bits.
5897         * Note that these particular counters are aggregated over
5898         * all 16 DMA engines.
5899         */
5900         for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
5901                 if (status & (1ull << i))
5902                         incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
5903         }
5904 }
5905
5906 /*
5907  * CCE block SDMA error interrupt.  Source is < 16.
5908  */
5909 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
5910 {
5911 #ifdef CONFIG_SDMA_VERBOSITY
5912         struct sdma_engine *sde = &dd->per_sdma[source];
5913
5914         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
5915                    slashstrip(__FILE__), __LINE__, __func__);
5916         dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
5917                    source);
5918         sdma_dumpstate(sde);
5919 #endif
5920         interrupt_clear_down(dd, source, &sdma_eng_err);
5921 }
5922
5923 /*
5924  * CCE block "various" interrupt.  Source is < 8.
5925  */
5926 static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
5927 {
5928         const struct err_reg_info *eri = &various_err[source];
5929
5930         /*
5931          * TCritInt cannot go through interrupt_clear_down()
5932          * because it is not a second tier interrupt. The handler
5933          * should be called directly.
5934          */
5935         if (source == TCRIT_INT_SOURCE)
5936                 handle_temp_err(dd);
5937         else if (eri->handler)
5938                 interrupt_clear_down(dd, 0, eri);
5939         else
5940                 dd_dev_info(dd,
5941                             "%s: Unimplemented/reserved interrupt %d\n",
5942                             __func__, source);
5943 }
5944
5945 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
5946 {
5947         /* src_ctx is always zero */
5948         struct hfi1_pportdata *ppd = dd->pport;
5949         unsigned long flags;
5950         u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
5951
5952         if (reg & QSFP_HFI0_MODPRST_N) {
5953                 if (!qsfp_mod_present(ppd)) {
5954                         dd_dev_info(dd, "%s: QSFP module removed\n",
5955                                     __func__);
5956
5957                         ppd->driver_link_ready = 0;
5958                         /*
5959                          * Cable removed, reset all our information about the
5960                          * cache and cable capabilities
5961                          */
5962
5963                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
5964                         /*
5965                          * We don't set cache_refresh_required here as we expect
5966                          * an interrupt when a cable is inserted
5967                          */
5968                         ppd->qsfp_info.cache_valid = 0;
5969                         ppd->qsfp_info.reset_needed = 0;
5970                         ppd->qsfp_info.limiting_active = 0;
5971                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
5972                                                flags);
5973                         /* Invert the ModPresent pin now to detect plug-in */
5974                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
5975                                   ASIC_QSFP1_INVERT, qsfp_int_mgmt);
5976
5977                         if ((ppd->offline_disabled_reason >
5978                           HFI1_ODR_MASK(
5979                           OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
5980                           (ppd->offline_disabled_reason ==
5981                           HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
5982                                 ppd->offline_disabled_reason =
5983                                 HFI1_ODR_MASK(
5984                                 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
5985
5986                         if (ppd->host_link_state == HLS_DN_POLL) {
5987                                 /*
5988                                  * The link is still in POLL. This means
5989                                  * that the normal link down processing
5990                                  * will not happen. We have to do it here
5991                                  * before turning the DC off.
5992                                  */
5993                                 queue_work(ppd->hfi1_wq, &ppd->link_down_work);
5994                         }
5995                 } else {
5996                         dd_dev_info(dd, "%s: QSFP module inserted\n",
5997                                     __func__);
5998
5999                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6000                         ppd->qsfp_info.cache_valid = 0;
6001                         ppd->qsfp_info.cache_refresh_required = 1;
6002                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6003                                                flags);
6004
6005                         /*
6006                          * Stop inversion of ModPresent pin to detect
6007                          * removal of the cable
6008                          */
6009                         qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
6010                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6011                                   ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6012
6013                         ppd->offline_disabled_reason =
6014                                 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
6015                 }
6016         }
6017
6018         if (reg & QSFP_HFI0_INT_N) {
6019                 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
6020                             __func__);
6021                 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6022                 ppd->qsfp_info.check_interrupt_flags = 1;
6023                 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6024         }
6025
6026         /* Schedule the QSFP work only if there is a cable attached. */
6027         if (qsfp_mod_present(ppd))
6028                 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
6029 }
6030
6031 static int request_host_lcb_access(struct hfi1_devdata *dd)
6032 {
6033         int ret;
6034
6035         ret = do_8051_command(dd, HCMD_MISC,
6036                               (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6037                               LOAD_DATA_FIELD_ID_SHIFT, NULL);
6038         if (ret != HCMD_SUCCESS) {
6039                 dd_dev_err(dd, "%s: command failed with error %d\n",
6040                            __func__, ret);
6041         }
6042         return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6043 }
6044
6045 static int request_8051_lcb_access(struct hfi1_devdata *dd)
6046 {
6047         int ret;
6048
6049         ret = do_8051_command(dd, HCMD_MISC,
6050                               (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6051                               LOAD_DATA_FIELD_ID_SHIFT, NULL);
6052         if (ret != HCMD_SUCCESS) {
6053                 dd_dev_err(dd, "%s: command failed with error %d\n",
6054                            __func__, ret);
6055         }
6056         return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6057 }
6058
6059 /*
6060  * Set the LCB selector - allow host access.  The DCC selector always
6061  * points to the host.
6062  */
6063 static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6064 {
6065         write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6066                   DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6067                   DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
6068 }
6069
6070 /*
6071  * Clear the LCB selector - allow 8051 access.  The DCC selector always
6072  * points to the host.
6073  */
6074 static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6075 {
6076         write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6077                   DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
6078 }
6079
6080 /*
6081  * Acquire LCB access from the 8051.  If the host already has access,
6082  * just increment a counter.  Otherwise, inform the 8051 that the
6083  * host is taking access.
6084  *
6085  * Returns:
6086  *      0 on success
6087  *      -EBUSY if the 8051 has control and cannot be disturbed
6088  *      -errno if unable to acquire access from the 8051
6089  */
6090 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6091 {
6092         struct hfi1_pportdata *ppd = dd->pport;
6093         int ret = 0;
6094
6095         /*
6096          * Use the host link state lock so the operation of this routine
6097          * { link state check, selector change, count increment } can occur
6098          * as a unit against a link state change.  Otherwise there is a
6099          * race between the state change and the count increment.
6100          */
6101         if (sleep_ok) {
6102                 mutex_lock(&ppd->hls_lock);
6103         } else {
6104                 while (!mutex_trylock(&ppd->hls_lock))
6105                         udelay(1);
6106         }
6107
6108         /* this access is valid only when the link is up */
6109         if (ppd->host_link_state & HLS_DOWN) {
6110                 dd_dev_info(dd, "%s: link state %s not up\n",
6111                             __func__, link_state_name(ppd->host_link_state));
6112                 ret = -EBUSY;
6113                 goto done;
6114         }
6115
6116         if (dd->lcb_access_count == 0) {
6117                 ret = request_host_lcb_access(dd);
6118                 if (ret) {
6119                         dd_dev_err(dd,
6120                                    "%s: unable to acquire LCB access, err %d\n",
6121                                    __func__, ret);
6122                         goto done;
6123                 }
6124                 set_host_lcb_access(dd);
6125         }
6126         dd->lcb_access_count++;
6127 done:
6128         mutex_unlock(&ppd->hls_lock);
6129         return ret;
6130 }
6131
6132 /*
6133  * Release LCB access by decrementing the use count.  If the count is moving
6134  * from 1 to 0, inform 8051 that it has control back.
6135  *
6136  * Returns:
6137  *      0 on success
6138  *      -errno if unable to release access to the 8051
6139  */
6140 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6141 {
6142         int ret = 0;
6143
6144         /*
6145          * Use the host link state lock because the acquire needed it.
6146          * Here, we only need to keep { selector change, count decrement }
6147          * as a unit.
6148          */
6149         if (sleep_ok) {
6150                 mutex_lock(&dd->pport->hls_lock);
6151         } else {
6152                 while (!mutex_trylock(&dd->pport->hls_lock))
6153                         udelay(1);
6154         }
6155
6156         if (dd->lcb_access_count == 0) {
6157                 dd_dev_err(dd, "%s: LCB access count is zero.  Skipping.\n",
6158                            __func__);
6159                 goto done;
6160         }
6161
6162         if (dd->lcb_access_count == 1) {
6163                 set_8051_lcb_access(dd);
6164                 ret = request_8051_lcb_access(dd);
6165                 if (ret) {
6166                         dd_dev_err(dd,
6167                                    "%s: unable to release LCB access, err %d\n",
6168                                    __func__, ret);
6169                         /* restore host access if the grant didn't work */
6170                         set_host_lcb_access(dd);
6171                         goto done;
6172                 }
6173         }
6174         dd->lcb_access_count--;
6175 done:
6176         mutex_unlock(&dd->pport->hls_lock);
6177         return ret;
6178 }
6179
6180 /*
6181  * Initialize LCB access variables and state.  Called during driver load,
6182  * after most of the initialization is finished.
6183  *
6184  * The DC default is LCB access on for the host.  The driver defaults to
6185  * leaving access to the 8051.  Assign access now - this constrains the call
6186  * to this routine to be after all LCB set-up is done.  In particular, after
6187  * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6188  */
6189 static void init_lcb_access(struct hfi1_devdata *dd)
6190 {
6191         dd->lcb_access_count = 0;
6192 }
6193
6194 /*
6195  * Write a response back to a 8051 request.
6196  */
6197 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6198 {
6199         write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
6200                   DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6201                   (u64)return_code <<
6202                   DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6203                   (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
6204 }
6205
6206 /*
6207  * Handle host requests from the 8051.
6208  */
6209 static void handle_8051_request(struct hfi1_pportdata *ppd)
6210 {
6211         struct hfi1_devdata *dd = ppd->dd;
6212         u64 reg;
6213         u16 data = 0;
6214         u8 type;
6215
6216         reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6217         if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6218                 return; /* no request */
6219
6220         /* zero out COMPLETED so the response is seen */
6221         write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6222
6223         /* extract request details */
6224         type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6225                         & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6226         data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6227                         & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6228
6229         switch (type) {
6230         case HREQ_LOAD_CONFIG:
6231         case HREQ_SAVE_CONFIG:
6232         case HREQ_READ_CONFIG:
6233         case HREQ_SET_TX_EQ_ABS:
6234         case HREQ_SET_TX_EQ_REL:
6235         case HREQ_ENABLE:
6236                 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
6237                             type);
6238                 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6239                 break;
6240         case HREQ_CONFIG_DONE:
6241                 hreq_response(dd, HREQ_SUCCESS, 0);
6242                 break;
6243
6244         case HREQ_INTERFACE_TEST:
6245                 hreq_response(dd, HREQ_SUCCESS, data);
6246                 break;
6247         default:
6248                 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6249                 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6250                 break;
6251         }
6252 }
6253
6254 static void write_global_credit(struct hfi1_devdata *dd,
6255                                 u8 vau, u16 total, u16 shared)
6256 {
6257         write_csr(dd, SEND_CM_GLOBAL_CREDIT,
6258                   ((u64)total <<
6259                    SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) |
6260                   ((u64)shared <<
6261                    SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) |
6262                   ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT));
6263 }
6264
6265 /*
6266  * Set up initial VL15 credits of the remote.  Assumes the rest of
6267  * the CM credit registers are zero from a previous global or credit reset .
6268  */
6269 void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf)
6270 {
6271         /* leave shared count at zero for both global and VL15 */
6272         write_global_credit(dd, vau, vl15buf, 0);
6273
6274         /* We may need some credits for another VL when sending packets
6275          * with the snoop interface. Dividing it down the middle for VL15
6276          * and VL0 should suffice.
6277          */
6278         if (unlikely(dd->hfi1_snoop.mode_flag == HFI1_PORT_SNOOP_MODE)) {
6279                 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)(vl15buf >> 1)
6280                     << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6281                 write_csr(dd, SEND_CM_CREDIT_VL, (u64)(vl15buf >> 1)
6282                     << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT);
6283         } else {
6284                 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6285                         << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6286         }
6287 }
6288
6289 /*
6290  * Zero all credit details from the previous connection and
6291  * reset the CM manager's internal counters.
6292  */
6293 void reset_link_credits(struct hfi1_devdata *dd)
6294 {
6295         int i;
6296
6297         /* remove all previous VL credit limits */
6298         for (i = 0; i < TXE_NUM_DATA_VL; i++)
6299                 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
6300         write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6301         write_global_credit(dd, 0, 0, 0);
6302         /* reset the CM block */
6303         pio_send_control(dd, PSC_CM_RESET);
6304 }
6305
6306 /* convert a vCU to a CU */
6307 static u32 vcu_to_cu(u8 vcu)
6308 {
6309         return 1 << vcu;
6310 }
6311
6312 /* convert a CU to a vCU */
6313 static u8 cu_to_vcu(u32 cu)
6314 {
6315         return ilog2(cu);
6316 }
6317
6318 /* convert a vAU to an AU */
6319 static u32 vau_to_au(u8 vau)
6320 {
6321         return 8 * (1 << vau);
6322 }
6323
6324 static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6325 {
6326         ppd->sm_trap_qp = 0x0;
6327         ppd->sa_qp = 0x1;
6328 }
6329
6330 /*
6331  * Graceful LCB shutdown.  This leaves the LCB FIFOs in reset.
6332  */
6333 static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6334 {
6335         u64 reg;
6336
6337         /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6338         write_csr(dd, DC_LCB_CFG_RUN, 0);
6339         /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6340         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
6341                   1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
6342         /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6343         dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6344         reg = read_csr(dd, DCC_CFG_RESET);
6345         write_csr(dd, DCC_CFG_RESET, reg |
6346                   (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) |
6347                   (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT));
6348         (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
6349         if (!abort) {
6350                 udelay(1);    /* must hold for the longer of 16cclks or 20ns */
6351                 write_csr(dd, DCC_CFG_RESET, reg);
6352                 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6353         }
6354 }
6355
6356 /*
6357  * This routine should be called after the link has been transitioned to
6358  * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6359  * reset).
6360  *
6361  * The expectation is that the caller of this routine would have taken
6362  * care of properly transitioning the link into the correct state.
6363  */
6364 static void dc_shutdown(struct hfi1_devdata *dd)
6365 {
6366         unsigned long flags;
6367
6368         spin_lock_irqsave(&dd->dc8051_lock, flags);
6369         if (dd->dc_shutdown) {
6370                 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6371                 return;
6372         }
6373         dd->dc_shutdown = 1;
6374         spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6375         /* Shutdown the LCB */
6376         lcb_shutdown(dd, 1);
6377         /*
6378          * Going to OFFLINE would have causes the 8051 to put the
6379          * SerDes into reset already. Just need to shut down the 8051,
6380          * itself.
6381          */
6382         write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6383 }
6384
6385 /*
6386  * Calling this after the DC has been brought out of reset should not
6387  * do any damage.
6388  */
6389 static void dc_start(struct hfi1_devdata *dd)
6390 {
6391         unsigned long flags;
6392         int ret;
6393
6394         spin_lock_irqsave(&dd->dc8051_lock, flags);
6395         if (!dd->dc_shutdown)
6396                 goto done;
6397         spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6398         /* Take the 8051 out of reset */
6399         write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6400         /* Wait until 8051 is ready */
6401         ret = wait_fm_ready(dd, TIMEOUT_8051_START);
6402         if (ret) {
6403                 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
6404                            __func__);
6405         }
6406         /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6407         write_csr(dd, DCC_CFG_RESET, 0x10);
6408         /* lcb_shutdown() with abort=1 does not restore these */
6409         write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6410         spin_lock_irqsave(&dd->dc8051_lock, flags);
6411         dd->dc_shutdown = 0;
6412 done:
6413         spin_unlock_irqrestore(&dd->dc8051_lock, flags);
6414 }
6415
6416 /*
6417  * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6418  */
6419 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6420 {
6421         u64 rx_radr, tx_radr;
6422         u32 version;
6423
6424         if (dd->icode != ICODE_FPGA_EMULATION)
6425                 return;
6426
6427         /*
6428          * These LCB defaults on emulator _s are good, nothing to do here:
6429          *      LCB_CFG_TX_FIFOS_RADR
6430          *      LCB_CFG_RX_FIFOS_RADR
6431          *      LCB_CFG_LN_DCLK
6432          *      LCB_CFG_IGNORE_LOST_RCLK
6433          */
6434         if (is_emulator_s(dd))
6435                 return;
6436         /* else this is _p */
6437
6438         version = emulator_rev(dd);
6439         if (!is_ax(dd))
6440                 version = 0x2d; /* all B0 use 0x2d or higher settings */
6441
6442         if (version <= 0x12) {
6443                 /* release 0x12 and below */
6444
6445                 /*
6446                  * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6447                  * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6448                  * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6449                  */
6450                 rx_radr =
6451                       0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6452                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6453                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6454                 /*
6455                  * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6456                  * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6457                  */
6458                 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6459         } else if (version <= 0x18) {
6460                 /* release 0x13 up to 0x18 */
6461                 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6462                 rx_radr =
6463                       0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6464                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6465                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6466                 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6467         } else if (version == 0x19) {
6468                 /* release 0x19 */
6469                 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6470                 rx_radr =
6471                       0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6472                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6473                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6474                 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6475         } else if (version == 0x1a) {
6476                 /* release 0x1a */
6477                 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6478                 rx_radr =
6479                       0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6480                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6481                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6482                 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6483                 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6484         } else {
6485                 /* release 0x1b and higher */
6486                 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6487                 rx_radr =
6488                       0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6489                     | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6490                     | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6491                 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6492         }
6493
6494         write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6495         /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6496         write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
6497                   DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
6498         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6499 }
6500
6501 /*
6502  * Handle a SMA idle message
6503  *
6504  * This is a work-queue function outside of the interrupt.
6505  */
6506 void handle_sma_message(struct work_struct *work)
6507 {
6508         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6509                                                         sma_message_work);
6510         struct hfi1_devdata *dd = ppd->dd;
6511         u64 msg;
6512         int ret;
6513
6514         /*
6515          * msg is bytes 1-4 of the 40-bit idle message - the command code
6516          * is stripped off
6517          */
6518         ret = read_idle_sma(dd, &msg);
6519         if (ret)
6520                 return;
6521         dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6522         /*
6523          * React to the SMA message.  Byte[1] (0 for us) is the command.
6524          */
6525         switch (msg & 0xff) {
6526         case SMA_IDLE_ARM:
6527                 /*
6528                  * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6529                  * State Transitions
6530                  *
6531                  * Only expected in INIT or ARMED, discard otherwise.
6532                  */
6533                 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6534                         ppd->neighbor_normal = 1;
6535                 break;
6536         case SMA_IDLE_ACTIVE:
6537                 /*
6538                  * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6539                  * State Transitions
6540                  *
6541                  * Can activate the node.  Discard otherwise.
6542                  */
6543                 if (ppd->host_link_state == HLS_UP_ARMED &&
6544                     ppd->is_active_optimize_enabled) {
6545                         ppd->neighbor_normal = 1;
6546                         ret = set_link_state(ppd, HLS_UP_ACTIVE);
6547                         if (ret)
6548                                 dd_dev_err(
6549                                         dd,
6550                                         "%s: received Active SMA idle message, couldn't set link to Active\n",
6551                                         __func__);
6552                 }
6553                 break;
6554         default:
6555                 dd_dev_err(dd,
6556                            "%s: received unexpected SMA idle message 0x%llx\n",
6557                            __func__, msg);
6558                 break;
6559         }
6560 }
6561
6562 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6563 {
6564         u64 rcvctrl;
6565         unsigned long flags;
6566
6567         spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6568         rcvctrl = read_csr(dd, RCV_CTRL);
6569         rcvctrl |= add;
6570         rcvctrl &= ~clear;
6571         write_csr(dd, RCV_CTRL, rcvctrl);
6572         spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6573 }
6574
6575 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6576 {
6577         adjust_rcvctrl(dd, add, 0);
6578 }
6579
6580 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6581 {
6582         adjust_rcvctrl(dd, 0, clear);
6583 }
6584
6585 /*
6586  * Called from all interrupt handlers to start handling an SPC freeze.
6587  */
6588 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6589 {
6590         struct hfi1_devdata *dd = ppd->dd;
6591         struct send_context *sc;
6592         int i;
6593
6594         if (flags & FREEZE_SELF)
6595                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6596
6597         /* enter frozen mode */
6598         dd->flags |= HFI1_FROZEN;
6599
6600         /* notify all SDMA engines that they are going into a freeze */
6601         sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6602
6603         /* do halt pre-handling on all enabled send contexts */
6604         for (i = 0; i < dd->num_send_contexts; i++) {
6605                 sc = dd->send_contexts[i].sc;
6606                 if (sc && (sc->flags & SCF_ENABLED))
6607                         sc_stop(sc, SCF_FROZEN | SCF_HALTED);
6608         }
6609
6610         /* Send context are frozen. Notify user space */
6611         hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6612
6613         if (flags & FREEZE_ABORT) {
6614                 dd_dev_err(dd,
6615                            "Aborted freeze recovery. Please REBOOT system\n");
6616                 return;
6617         }
6618         /* queue non-interrupt handler */
6619         queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6620 }
6621
6622 /*
6623  * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6624  * depending on the "freeze" parameter.
6625  *
6626  * No need to return an error if it times out, our only option
6627  * is to proceed anyway.
6628  */
6629 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6630 {
6631         unsigned long timeout;
6632         u64 reg;
6633
6634         timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6635         while (1) {
6636                 reg = read_csr(dd, CCE_STATUS);
6637                 if (freeze) {
6638                         /* waiting until all indicators are set */
6639                         if ((reg & ALL_FROZE) == ALL_FROZE)
6640                                 return; /* all done */
6641                 } else {
6642                         /* waiting until all indicators are clear */
6643                         if ((reg & ALL_FROZE) == 0)
6644                                 return; /* all done */
6645                 }
6646
6647                 if (time_after(jiffies, timeout)) {
6648                         dd_dev_err(dd,
6649                                    "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6650                                    freeze ? "" : "un", reg & ALL_FROZE,
6651                                    freeze ? ALL_FROZE : 0ull);
6652                         return;
6653                 }
6654                 usleep_range(80, 120);
6655         }
6656 }
6657
6658 /*
6659  * Do all freeze handling for the RXE block.
6660  */
6661 static void rxe_freeze(struct hfi1_devdata *dd)
6662 {
6663         int i;
6664
6665         /* disable port */
6666         clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6667
6668         /* disable all receive contexts */
6669         for (i = 0; i < dd->num_rcv_contexts; i++)
6670                 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
6671 }
6672
6673 /*
6674  * Unfreeze handling for the RXE block - kernel contexts only.
6675  * This will also enable the port.  User contexts will do unfreeze
6676  * handling on a per-context basis as they call into the driver.
6677  *
6678  */
6679 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6680 {
6681         u32 rcvmask;
6682         int i;
6683
6684         /* enable all kernel contexts */
6685         for (i = 0; i < dd->n_krcv_queues; i++) {
6686                 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6687                 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6688                 rcvmask |= HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, DMA_RTAIL) ?
6689                         HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6690                 hfi1_rcvctrl(dd, rcvmask, i);
6691         }
6692
6693         /* enable port */
6694         add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6695 }
6696
6697 /*
6698  * Non-interrupt SPC freeze handling.
6699  *
6700  * This is a work-queue function outside of the triggering interrupt.
6701  */
6702 void handle_freeze(struct work_struct *work)
6703 {
6704         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6705                                                                 freeze_work);
6706         struct hfi1_devdata *dd = ppd->dd;
6707
6708         /* wait for freeze indicators on all affected blocks */
6709         wait_for_freeze_status(dd, 1);
6710
6711         /* SPC is now frozen */
6712
6713         /* do send PIO freeze steps */
6714         pio_freeze(dd);
6715
6716         /* do send DMA freeze steps */
6717         sdma_freeze(dd);
6718
6719         /* do send egress freeze steps - nothing to do */
6720
6721         /* do receive freeze steps */
6722         rxe_freeze(dd);
6723
6724         /*
6725          * Unfreeze the hardware - clear the freeze, wait for each
6726          * block's frozen bit to clear, then clear the frozen flag.
6727          */
6728         write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6729         wait_for_freeze_status(dd, 0);
6730
6731         if (is_ax(dd)) {
6732                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6733                 wait_for_freeze_status(dd, 1);
6734                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6735                 wait_for_freeze_status(dd, 0);
6736         }
6737
6738         /* do send PIO unfreeze steps for kernel contexts */
6739         pio_kernel_unfreeze(dd);
6740
6741         /* do send DMA unfreeze steps */
6742         sdma_unfreeze(dd);
6743
6744         /* do send egress unfreeze steps - nothing to do */
6745
6746         /* do receive unfreeze steps for kernel contexts */
6747         rxe_kernel_unfreeze(dd);
6748
6749         /*
6750          * The unfreeze procedure touches global device registers when
6751          * it disables and re-enables RXE. Mark the device unfrozen
6752          * after all that is done so other parts of the driver waiting
6753          * for the device to unfreeze don't do things out of order.
6754          *
6755          * The above implies that the meaning of HFI1_FROZEN flag is
6756          * "Device has gone into freeze mode and freeze mode handling
6757          * is still in progress."
6758          *
6759          * The flag will be removed when freeze mode processing has
6760          * completed.
6761          */
6762         dd->flags &= ~HFI1_FROZEN;
6763         wake_up(&dd->event_queue);
6764
6765         /* no longer frozen */
6766 }
6767
6768 /*
6769  * Handle a link up interrupt from the 8051.
6770  *
6771  * This is a work-queue function outside of the interrupt.
6772  */
6773 void handle_link_up(struct work_struct *work)
6774 {
6775         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6776                                                   link_up_work);
6777         set_link_state(ppd, HLS_UP_INIT);
6778
6779         /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6780         read_ltp_rtt(ppd->dd);
6781         /*
6782          * OPA specifies that certain counters are cleared on a transition
6783          * to link up, so do that.
6784          */
6785         clear_linkup_counters(ppd->dd);
6786         /*
6787          * And (re)set link up default values.
6788          */
6789         set_linkup_defaults(ppd);
6790
6791         /* enforce link speed enabled */
6792         if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6793                 /* oops - current speed is not enabled, bounce */
6794                 dd_dev_err(ppd->dd,
6795                            "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6796                            ppd->link_speed_active, ppd->link_speed_enabled);
6797                 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
6798                                      OPA_LINKDOWN_REASON_SPEED_POLICY);
6799                 set_link_state(ppd, HLS_DN_OFFLINE);
6800                 tune_serdes(ppd);
6801                 start_link(ppd);
6802         }
6803 }
6804
6805 /*
6806  * Several pieces of LNI information were cached for SMA in ppd.
6807  * Reset these on link down
6808  */
6809 static void reset_neighbor_info(struct hfi1_pportdata *ppd)
6810 {
6811         ppd->neighbor_guid = 0;
6812         ppd->neighbor_port_number = 0;
6813         ppd->neighbor_type = 0;
6814         ppd->neighbor_fm_security = 0;
6815 }
6816
6817 static const char * const link_down_reason_strs[] = {
6818         [OPA_LINKDOWN_REASON_NONE] = "None",
6819         [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Recive error 0",
6820         [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
6821         [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
6822         [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
6823         [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
6824         [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
6825         [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
6826         [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
6827         [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
6828         [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
6829         [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
6830         [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
6831         [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
6832         [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
6833         [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
6834         [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
6835         [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
6836         [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
6837         [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
6838         [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
6839         [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
6840         [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
6841         [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
6842         [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
6843         [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
6844         [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
6845         [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
6846         [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
6847         [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
6848         [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
6849         [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
6850         [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
6851                                         "Excessive buffer overrun",
6852         [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
6853         [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
6854         [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
6855         [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
6856         [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
6857         [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
6858         [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
6859         [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
6860                                         "Local media not installed",
6861         [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
6862         [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
6863         [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
6864                                         "End to end not installed",
6865         [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
6866         [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
6867         [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
6868         [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
6869         [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
6870         [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
6871 };
6872
6873 /* return the neighbor link down reason string */
6874 static const char *link_down_reason_str(u8 reason)
6875 {
6876         const char *str = NULL;
6877
6878         if (reason < ARRAY_SIZE(link_down_reason_strs))
6879                 str = link_down_reason_strs[reason];
6880         if (!str)
6881                 str = "(invalid)";
6882
6883         return str;
6884 }
6885
6886 /*
6887  * Handle a link down interrupt from the 8051.
6888  *
6889  * This is a work-queue function outside of the interrupt.
6890  */
6891 void handle_link_down(struct work_struct *work)
6892 {
6893         u8 lcl_reason, neigh_reason = 0;
6894         u8 link_down_reason;
6895         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6896                                                   link_down_work);
6897         int was_up;
6898         static const char ldr_str[] = "Link down reason: ";
6899
6900         if ((ppd->host_link_state &
6901              (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
6902              ppd->port_type == PORT_TYPE_FIXED)
6903                 ppd->offline_disabled_reason =
6904                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
6905
6906         /* Go offline first, then deal with reading/writing through 8051 */
6907         was_up = !!(ppd->host_link_state & HLS_UP);
6908         set_link_state(ppd, HLS_DN_OFFLINE);
6909
6910         if (was_up) {
6911                 lcl_reason = 0;
6912                 /* link down reason is only valid if the link was up */
6913                 read_link_down_reason(ppd->dd, &link_down_reason);
6914                 switch (link_down_reason) {
6915                 case LDR_LINK_TRANSFER_ACTIVE_LOW:
6916                         /* the link went down, no idle message reason */
6917                         dd_dev_info(ppd->dd, "%sUnexpected link down\n",
6918                                     ldr_str);
6919                         break;
6920                 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
6921                         /*
6922                          * The neighbor reason is only valid if an idle message
6923                          * was received for it.
6924                          */
6925                         read_planned_down_reason_code(ppd->dd, &neigh_reason);
6926                         dd_dev_info(ppd->dd,
6927                                     "%sNeighbor link down message %d, %s\n",
6928                                     ldr_str, neigh_reason,
6929                                     link_down_reason_str(neigh_reason));
6930                         break;
6931                 case LDR_RECEIVED_HOST_OFFLINE_REQ:
6932                         dd_dev_info(ppd->dd,
6933                                     "%sHost requested link to go offline\n",
6934                                     ldr_str);
6935                         break;
6936                 default:
6937                         dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
6938                                     ldr_str, link_down_reason);
6939                         break;
6940                 }
6941
6942                 /*
6943                  * If no reason, assume peer-initiated but missed
6944                  * LinkGoingDown idle flits.
6945                  */
6946                 if (neigh_reason == 0)
6947                         lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
6948         } else {
6949                 /* went down while polling or going up */
6950                 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
6951         }
6952
6953         set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
6954
6955         /* inform the SMA when the link transitions from up to down */
6956         if (was_up && ppd->local_link_down_reason.sma == 0 &&
6957             ppd->neigh_link_down_reason.sma == 0) {
6958                 ppd->local_link_down_reason.sma =
6959                                         ppd->local_link_down_reason.latest;
6960                 ppd->neigh_link_down_reason.sma =
6961                                         ppd->neigh_link_down_reason.latest;
6962         }
6963
6964         reset_neighbor_info(ppd);
6965         if (ppd->mgmt_allowed)
6966                 remove_full_mgmt_pkey(ppd);
6967
6968         /* disable the port */
6969         clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6970
6971         /*
6972          * If there is no cable attached, turn the DC off. Otherwise,
6973          * start the link bring up.
6974          */
6975         if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) {
6976                 dc_shutdown(ppd->dd);
6977         } else {
6978                 tune_serdes(ppd);
6979                 start_link(ppd);
6980         }
6981 }
6982
6983 void handle_link_bounce(struct work_struct *work)
6984 {
6985         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6986                                                         link_bounce_work);
6987
6988         /*
6989          * Only do something if the link is currently up.
6990          */
6991         if (ppd->host_link_state & HLS_UP) {
6992                 set_link_state(ppd, HLS_DN_OFFLINE);
6993                 tune_serdes(ppd);
6994                 start_link(ppd);
6995         } else {
6996                 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
6997                             __func__, link_state_name(ppd->host_link_state));
6998         }
6999 }
7000
7001 /*
7002  * Mask conversion: Capability exchange to Port LTP.  The capability
7003  * exchange has an implicit 16b CRC that is mandatory.
7004  */
7005 static int cap_to_port_ltp(int cap)
7006 {
7007         int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
7008
7009         if (cap & CAP_CRC_14B)
7010                 port_ltp |= PORT_LTP_CRC_MODE_14;
7011         if (cap & CAP_CRC_48B)
7012                 port_ltp |= PORT_LTP_CRC_MODE_48;
7013         if (cap & CAP_CRC_12B_16B_PER_LANE)
7014                 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7015
7016         return port_ltp;
7017 }
7018
7019 /*
7020  * Convert an OPA Port LTP mask to capability mask
7021  */
7022 int port_ltp_to_cap(int port_ltp)
7023 {
7024         int cap_mask = 0;
7025
7026         if (port_ltp & PORT_LTP_CRC_MODE_14)
7027                 cap_mask |= CAP_CRC_14B;
7028         if (port_ltp & PORT_LTP_CRC_MODE_48)
7029                 cap_mask |= CAP_CRC_48B;
7030         if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7031                 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7032
7033         return cap_mask;
7034 }
7035
7036 /*
7037  * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
7038  */
7039 static int lcb_to_port_ltp(int lcb_crc)
7040 {
7041         int port_ltp = 0;
7042
7043         if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7044                 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7045         else if (lcb_crc == LCB_CRC_48B)
7046                 port_ltp = PORT_LTP_CRC_MODE_48;
7047         else if (lcb_crc == LCB_CRC_14B)
7048                 port_ltp = PORT_LTP_CRC_MODE_14;
7049         else
7050                 port_ltp = PORT_LTP_CRC_MODE_16;
7051
7052         return port_ltp;
7053 }
7054
7055 /*
7056  * Our neighbor has indicated that we are allowed to act as a fabric
7057  * manager, so place the full management partition key in the second
7058  * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note
7059  * that we should already have the limited management partition key in
7060  * array element 1, and also that the port is not yet up when
7061  * add_full_mgmt_pkey() is invoked.
7062  */
7063 static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7064 {
7065         struct hfi1_devdata *dd = ppd->dd;
7066
7067         /* Sanity check - ppd->pkeys[2] should be 0, or already initalized */
7068         if (!((ppd->pkeys[2] == 0) || (ppd->pkeys[2] == FULL_MGMT_P_KEY)))
7069                 dd_dev_warn(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n",
7070                             __func__, ppd->pkeys[2], FULL_MGMT_P_KEY);
7071         ppd->pkeys[2] = FULL_MGMT_P_KEY;
7072         (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7073 }
7074
7075 static void remove_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7076 {
7077         ppd->pkeys[2] = 0;
7078         (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7079 }
7080
7081 /*
7082  * Convert the given link width to the OPA link width bitmask.
7083  */
7084 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7085 {
7086         switch (width) {
7087         case 0:
7088                 /*
7089                  * Simulator and quick linkup do not set the width.
7090                  * Just set it to 4x without complaint.
7091                  */
7092                 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7093                         return OPA_LINK_WIDTH_4X;
7094                 return 0; /* no lanes up */
7095         case 1: return OPA_LINK_WIDTH_1X;
7096         case 2: return OPA_LINK_WIDTH_2X;
7097         case 3: return OPA_LINK_WIDTH_3X;
7098         default:
7099                 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
7100                             __func__, width);
7101                 /* fall through */
7102         case 4: return OPA_LINK_WIDTH_4X;
7103         }
7104 }
7105
7106 /*
7107  * Do a population count on the bottom nibble.
7108  */
7109 static const u8 bit_counts[16] = {
7110         0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7111 };
7112
7113 static inline u8 nibble_to_count(u8 nibble)
7114 {
7115         return bit_counts[nibble & 0xf];
7116 }
7117
7118 /*
7119  * Read the active lane information from the 8051 registers and return
7120  * their widths.
7121  *
7122  * Active lane information is found in these 8051 registers:
7123  *      enable_lane_tx
7124  *      enable_lane_rx
7125  */
7126 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7127                             u16 *rx_width)
7128 {
7129         u16 tx, rx;
7130         u8 enable_lane_rx;
7131         u8 enable_lane_tx;
7132         u8 tx_polarity_inversion;
7133         u8 rx_polarity_inversion;
7134         u8 max_rate;
7135
7136         /* read the active lanes */
7137         read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
7138                          &rx_polarity_inversion, &max_rate);
7139         read_local_lni(dd, &enable_lane_rx);
7140
7141         /* convert to counts */
7142         tx = nibble_to_count(enable_lane_tx);
7143         rx = nibble_to_count(enable_lane_rx);
7144
7145         /*
7146          * Set link_speed_active here, overriding what was set in
7147          * handle_verify_cap().  The ASIC 8051 firmware does not correctly
7148          * set the max_rate field in handle_verify_cap until v0.19.
7149          */
7150         if ((dd->icode == ICODE_RTL_SILICON) &&
7151             (dd->dc8051_ver < dc8051_ver(0, 19))) {
7152                 /* max_rate: 0 = 12.5G, 1 = 25G */
7153                 switch (max_rate) {
7154                 case 0:
7155                         dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7156                         break;
7157                 default:
7158                         dd_dev_err(dd,
7159                                    "%s: unexpected max rate %d, using 25Gb\n",
7160                                    __func__, (int)max_rate);
7161                         /* fall through */
7162                 case 1:
7163                         dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7164                         break;
7165                 }
7166         }
7167
7168         dd_dev_info(dd,
7169                     "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7170                     enable_lane_tx, tx, enable_lane_rx, rx);
7171         *tx_width = link_width_to_bits(dd, tx);
7172         *rx_width = link_width_to_bits(dd, rx);
7173 }
7174
7175 /*
7176  * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7177  * Valid after the end of VerifyCap and during LinkUp.  Does not change
7178  * after link up.  I.e. look elsewhere for downgrade information.
7179  *
7180  * Bits are:
7181  *      + bits [7:4] contain the number of active transmitters
7182  *      + bits [3:0] contain the number of active receivers
7183  * These are numbers 1 through 4 and can be different values if the
7184  * link is asymmetric.
7185  *
7186  * verify_cap_local_fm_link_width[0] retains its original value.
7187  */
7188 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7189                               u16 *rx_width)
7190 {
7191         u16 widths, tx, rx;
7192         u8 misc_bits, local_flags;
7193         u16 active_tx, active_rx;
7194
7195         read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths);
7196         tx = widths >> 12;
7197         rx = (widths >> 8) & 0xf;
7198
7199         *tx_width = link_width_to_bits(dd, tx);
7200         *rx_width = link_width_to_bits(dd, rx);
7201
7202         /* print the active widths */
7203         get_link_widths(dd, &active_tx, &active_rx);
7204 }
7205
7206 /*
7207  * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7208  * hardware information when the link first comes up.
7209  *
7210  * The link width is not available until after VerifyCap.AllFramesReceived
7211  * (the trigger for handle_verify_cap), so this is outside that routine
7212  * and should be called when the 8051 signals linkup.
7213  */
7214 void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7215 {
7216         u16 tx_width, rx_width;
7217
7218         /* get end-of-LNI link widths */
7219         get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7220
7221         /* use tx_width as the link is supposed to be symmetric on link up */
7222         ppd->link_width_active = tx_width;
7223         /* link width downgrade active (LWD.A) starts out matching LW.A */
7224         ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7225         ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7226         /* per OPA spec, on link up LWD.E resets to LWD.S */
7227         ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7228         /* cache the active egress rate (units {10^6 bits/sec]) */
7229         ppd->current_egress_rate = active_egress_rate(ppd);
7230 }
7231
7232 /*
7233  * Handle a verify capabilities interrupt from the 8051.
7234  *
7235  * This is a work-queue function outside of the interrupt.
7236  */
7237 void handle_verify_cap(struct work_struct *work)
7238 {
7239         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7240                                                                 link_vc_work);
7241         struct hfi1_devdata *dd = ppd->dd;
7242         u64 reg;
7243         u8 power_management;
7244         u8 continious;
7245         u8 vcu;
7246         u8 vau;
7247         u8 z;
7248         u16 vl15buf;
7249         u16 link_widths;
7250         u16 crc_mask;
7251         u16 crc_val;
7252         u16 device_id;
7253         u16 active_tx, active_rx;
7254         u8 partner_supported_crc;
7255         u8 remote_tx_rate;
7256         u8 device_rev;
7257
7258         set_link_state(ppd, HLS_VERIFY_CAP);
7259
7260         lcb_shutdown(dd, 0);
7261         adjust_lcb_for_fpga_serdes(dd);
7262
7263         /*
7264          * These are now valid:
7265          *      remote VerifyCap fields in the general LNI config
7266          *      CSR DC8051_STS_REMOTE_GUID
7267          *      CSR DC8051_STS_REMOTE_NODE_TYPE
7268          *      CSR DC8051_STS_REMOTE_FM_SECURITY
7269          *      CSR DC8051_STS_REMOTE_PORT_NO
7270          */
7271
7272         read_vc_remote_phy(dd, &power_management, &continious);
7273         read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7274                               &partner_supported_crc);
7275         read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7276         read_remote_device_id(dd, &device_id, &device_rev);
7277         /*
7278          * And the 'MgmtAllowed' information, which is exchanged during
7279          * LNI, is also be available at this point.
7280          */
7281         read_mgmt_allowed(dd, &ppd->mgmt_allowed);
7282         /* print the active widths */
7283         get_link_widths(dd, &active_tx, &active_rx);
7284         dd_dev_info(dd,
7285                     "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7286                     (int)power_management, (int)continious);
7287         dd_dev_info(dd,
7288                     "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7289                     (int)vau, (int)z, (int)vcu, (int)vl15buf,
7290                     (int)partner_supported_crc);
7291         dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
7292                     (u32)remote_tx_rate, (u32)link_widths);
7293         dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
7294                     (u32)device_id, (u32)device_rev);
7295         /*
7296          * The peer vAU value just read is the peer receiver value.  HFI does
7297          * not support a transmit vAU of 0 (AU == 8).  We advertised that
7298          * with Z=1 in the fabric capabilities sent to the peer.  The peer
7299          * will see our Z=1, and, if it advertised a vAU of 0, will move its
7300          * receive to vAU of 1 (AU == 16).  Do the same here.  We do not care
7301          * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7302          * subject to the Z value exception.
7303          */
7304         if (vau == 0)
7305                 vau = 1;
7306         set_up_vl15(dd, vau, vl15buf);
7307
7308         /* set up the LCB CRC mode */
7309         crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7310
7311         /* order is important: use the lowest bit in common */
7312         if (crc_mask & CAP_CRC_14B)
7313                 crc_val = LCB_CRC_14B;
7314         else if (crc_mask & CAP_CRC_48B)
7315                 crc_val = LCB_CRC_48B;
7316         else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7317                 crc_val = LCB_CRC_12B_16B_PER_LANE;
7318         else
7319                 crc_val = LCB_CRC_16B;
7320
7321         dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7322         write_csr(dd, DC_LCB_CFG_CRC_MODE,
7323                   (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7324
7325         /* set (14b only) or clear sideband credit */
7326         reg = read_csr(dd, SEND_CM_CTRL);
7327         if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7328                 write_csr(dd, SEND_CM_CTRL,
7329                           reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7330         } else {
7331                 write_csr(dd, SEND_CM_CTRL,
7332                           reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7333         }
7334
7335         ppd->link_speed_active = 0;     /* invalid value */
7336         if (dd->dc8051_ver < dc8051_ver(0, 20)) {
7337                 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7338                 switch (remote_tx_rate) {
7339                 case 0:
7340                         ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7341                         break;
7342                 case 1:
7343                         ppd->link_speed_active = OPA_LINK_SPEED_25G;
7344                         break;
7345                 }
7346         } else {
7347                 /* actual rate is highest bit of the ANDed rates */
7348                 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7349
7350                 if (rate & 2)
7351                         ppd->link_speed_active = OPA_LINK_SPEED_25G;
7352                 else if (rate & 1)
7353                         ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7354         }
7355         if (ppd->link_speed_active == 0) {
7356                 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
7357                            __func__, (int)remote_tx_rate);
7358                 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7359         }
7360
7361         /*
7362          * Cache the values of the supported, enabled, and active
7363          * LTP CRC modes to return in 'portinfo' queries. But the bit
7364          * flags that are returned in the portinfo query differ from
7365          * what's in the link_crc_mask, crc_sizes, and crc_val
7366          * variables. Convert these here.
7367          */
7368         ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7369                 /* supported crc modes */
7370         ppd->port_ltp_crc_mode |=
7371                 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7372                 /* enabled crc modes */
7373         ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7374                 /* active crc mode */
7375
7376         /* set up the remote credit return table */
7377         assign_remote_cm_au_table(dd, vcu);
7378
7379         /*
7380          * The LCB is reset on entry to handle_verify_cap(), so this must
7381          * be applied on every link up.
7382          *
7383          * Adjust LCB error kill enable to kill the link if
7384          * these RBUF errors are seen:
7385          *      REPLAY_BUF_MBE_SMASK
7386          *      FLIT_INPUT_BUF_MBE_SMASK
7387          */
7388         if (is_ax(dd)) {                        /* fixed in B0 */
7389                 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7390                 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7391                         | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7392                 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7393         }
7394
7395         /* pull LCB fifos out of reset - all fifo clocks must be stable */
7396         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7397
7398         /* give 8051 access to the LCB CSRs */
7399         write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7400         set_8051_lcb_access(dd);
7401
7402         ppd->neighbor_guid =
7403                 read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
7404         ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
7405                                         DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
7406         ppd->neighbor_type =
7407                 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
7408                 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
7409         ppd->neighbor_fm_security =
7410                 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
7411                 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
7412         dd_dev_info(dd,
7413                     "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
7414                     ppd->neighbor_guid, ppd->neighbor_type,
7415                     ppd->mgmt_allowed, ppd->neighbor_fm_security);
7416         if (ppd->mgmt_allowed)
7417                 add_full_mgmt_pkey(ppd);
7418
7419         /* tell the 8051 to go to LinkUp */
7420         set_link_state(ppd, HLS_GOING_UP);
7421 }
7422
7423 /*
7424  * Apply the link width downgrade enabled policy against the current active
7425  * link widths.
7426  *
7427  * Called when the enabled policy changes or the active link widths change.
7428  */
7429 void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
7430 {
7431         int do_bounce = 0;
7432         int tries;
7433         u16 lwde;
7434         u16 tx, rx;
7435
7436         /* use the hls lock to avoid a race with actual link up */
7437         tries = 0;
7438 retry:
7439         mutex_lock(&ppd->hls_lock);
7440         /* only apply if the link is up */
7441         if (ppd->host_link_state & HLS_DOWN) {
7442                 /* still going up..wait and retry */
7443                 if (ppd->host_link_state & HLS_GOING_UP) {
7444                         if (++tries < 1000) {
7445                                 mutex_unlock(&ppd->hls_lock);
7446                                 usleep_range(100, 120); /* arbitrary */
7447                                 goto retry;
7448                         }
7449                         dd_dev_err(ppd->dd,
7450                                    "%s: giving up waiting for link state change\n",
7451                                    __func__);
7452                 }
7453                 goto done;
7454         }
7455
7456         lwde = ppd->link_width_downgrade_enabled;
7457
7458         if (refresh_widths) {
7459                 get_link_widths(ppd->dd, &tx, &rx);
7460                 ppd->link_width_downgrade_tx_active = tx;
7461                 ppd->link_width_downgrade_rx_active = rx;
7462         }
7463
7464         if (ppd->link_width_downgrade_tx_active == 0 ||
7465             ppd->link_width_downgrade_rx_active == 0) {
7466                 /* the 8051 reported a dead link as a downgrade */
7467                 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7468         } else if (lwde == 0) {
7469                 /* downgrade is disabled */
7470
7471                 /* bounce if not at starting active width */
7472                 if ((ppd->link_width_active !=
7473                      ppd->link_width_downgrade_tx_active) ||
7474                     (ppd->link_width_active !=
7475                      ppd->link_width_downgrade_rx_active)) {
7476                         dd_dev_err(ppd->dd,
7477                                    "Link downgrade is disabled and link has downgraded, downing link\n");
7478                         dd_dev_err(ppd->dd,
7479                                    "  original 0x%x, tx active 0x%x, rx active 0x%x\n",
7480                                    ppd->link_width_active,
7481                                    ppd->link_width_downgrade_tx_active,
7482                                    ppd->link_width_downgrade_rx_active);
7483                         do_bounce = 1;
7484                 }
7485         } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7486                    (lwde & ppd->link_width_downgrade_rx_active) == 0) {
7487                 /* Tx or Rx is outside the enabled policy */
7488                 dd_dev_err(ppd->dd,
7489                            "Link is outside of downgrade allowed, downing link\n");
7490                 dd_dev_err(ppd->dd,
7491                            "  enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7492                            lwde, ppd->link_width_downgrade_tx_active,
7493                            ppd->link_width_downgrade_rx_active);
7494                 do_bounce = 1;
7495         }
7496
7497 done:
7498         mutex_unlock(&ppd->hls_lock);
7499
7500         if (do_bounce) {
7501                 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
7502                                      OPA_LINKDOWN_REASON_WIDTH_POLICY);
7503                 set_link_state(ppd, HLS_DN_OFFLINE);
7504                 tune_serdes(ppd);
7505                 start_link(ppd);
7506         }
7507 }
7508
7509 /*
7510  * Handle a link downgrade interrupt from the 8051.
7511  *
7512  * This is a work-queue function outside of the interrupt.
7513  */
7514 void handle_link_downgrade(struct work_struct *work)
7515 {
7516         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7517                                                         link_downgrade_work);
7518
7519         dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7520         apply_link_downgrade_policy(ppd, 1);
7521 }
7522
7523 static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7524 {
7525         return flag_string(buf, buf_len, flags, dcc_err_flags,
7526                 ARRAY_SIZE(dcc_err_flags));
7527 }
7528
7529 static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7530 {
7531         return flag_string(buf, buf_len, flags, lcb_err_flags,
7532                 ARRAY_SIZE(lcb_err_flags));
7533 }
7534
7535 static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7536 {
7537         return flag_string(buf, buf_len, flags, dc8051_err_flags,
7538                 ARRAY_SIZE(dc8051_err_flags));
7539 }
7540
7541 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7542 {
7543         return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7544                 ARRAY_SIZE(dc8051_info_err_flags));
7545 }
7546
7547 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7548 {
7549         return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7550                 ARRAY_SIZE(dc8051_info_host_msg_flags));
7551 }
7552
7553 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7554 {
7555         struct hfi1_pportdata *ppd = dd->pport;
7556         u64 info, err, host_msg;
7557         int queue_link_down = 0;
7558         char buf[96];
7559
7560         /* look at the flags */
7561         if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7562                 /* 8051 information set by firmware */
7563                 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7564                 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7565                 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7566                         & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7567                 host_msg = (info >>
7568                         DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7569                         & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7570
7571                 /*
7572                  * Handle error flags.
7573                  */
7574                 if (err & FAILED_LNI) {
7575                         /*
7576                          * LNI error indications are cleared by the 8051
7577                          * only when starting polling.  Only pay attention
7578                          * to them when in the states that occur during
7579                          * LNI.
7580                          */
7581                         if (ppd->host_link_state
7582                             & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7583                                 queue_link_down = 1;
7584                                 dd_dev_info(dd, "Link error: %s\n",
7585                                             dc8051_info_err_string(buf,
7586                                                                    sizeof(buf),
7587                                                                    err &
7588                                                                    FAILED_LNI));
7589                         }
7590                         err &= ~(u64)FAILED_LNI;
7591                 }
7592                 /* unknown frames can happen durning LNI, just count */
7593                 if (err & UNKNOWN_FRAME) {
7594                         ppd->unknown_frame_count++;
7595                         err &= ~(u64)UNKNOWN_FRAME;
7596                 }
7597                 if (err) {
7598                         /* report remaining errors, but do not do anything */
7599                         dd_dev_err(dd, "8051 info error: %s\n",
7600                                    dc8051_info_err_string(buf, sizeof(buf),
7601                                                           err));
7602                 }
7603
7604                 /*
7605                  * Handle host message flags.
7606                  */
7607                 if (host_msg & HOST_REQ_DONE) {
7608                         /*
7609                          * Presently, the driver does a busy wait for
7610                          * host requests to complete.  This is only an
7611                          * informational message.
7612                          * NOTE: The 8051 clears the host message
7613                          * information *on the next 8051 command*.
7614                          * Therefore, when linkup is achieved,
7615                          * this flag will still be set.
7616                          */
7617                         host_msg &= ~(u64)HOST_REQ_DONE;
7618                 }
7619                 if (host_msg & BC_SMA_MSG) {
7620                         queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
7621                         host_msg &= ~(u64)BC_SMA_MSG;
7622                 }
7623                 if (host_msg & LINKUP_ACHIEVED) {
7624                         dd_dev_info(dd, "8051: Link up\n");
7625                         queue_work(ppd->hfi1_wq, &ppd->link_up_work);
7626                         host_msg &= ~(u64)LINKUP_ACHIEVED;
7627                 }
7628                 if (host_msg & EXT_DEVICE_CFG_REQ) {
7629                         handle_8051_request(ppd);
7630                         host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7631                 }
7632                 if (host_msg & VERIFY_CAP_FRAME) {
7633                         queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
7634                         host_msg &= ~(u64)VERIFY_CAP_FRAME;
7635                 }
7636                 if (host_msg & LINK_GOING_DOWN) {
7637                         const char *extra = "";
7638                         /* no downgrade action needed if going down */
7639                         if (host_msg & LINK_WIDTH_DOWNGRADED) {
7640                                 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7641                                 extra = " (ignoring downgrade)";
7642                         }
7643                         dd_dev_info(dd, "8051: Link down%s\n", extra);
7644                         queue_link_down = 1;
7645                         host_msg &= ~(u64)LINK_GOING_DOWN;
7646                 }
7647                 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7648                         queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
7649                         host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7650                 }
7651                 if (host_msg) {
7652                         /* report remaining messages, but do not do anything */
7653                         dd_dev_info(dd, "8051 info host message: %s\n",
7654                                     dc8051_info_host_msg_string(buf,
7655                                                                 sizeof(buf),
7656                                                                 host_msg));
7657                 }
7658
7659                 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7660         }
7661         if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7662                 /*
7663                  * Lost the 8051 heartbeat.  If this happens, we
7664                  * receive constant interrupts about it.  Disable
7665                  * the interrupt after the first.
7666                  */
7667                 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7668                 write_csr(dd, DC_DC8051_ERR_EN,
7669                           read_csr(dd, DC_DC8051_ERR_EN) &
7670                           ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
7671
7672                 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7673         }
7674         if (reg) {
7675                 /* report the error, but do not do anything */
7676                 dd_dev_err(dd, "8051 error: %s\n",
7677                            dc8051_err_string(buf, sizeof(buf), reg));
7678         }
7679
7680         if (queue_link_down) {
7681                 /*
7682                  * if the link is already going down or disabled, do not
7683                  * queue another
7684                  */
7685                 if ((ppd->host_link_state &
7686                     (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7687                     ppd->link_enabled == 0) {
7688                         dd_dev_info(dd, "%s: not queuing link down\n",
7689                                     __func__);
7690                 } else {
7691                         queue_work(ppd->hfi1_wq, &ppd->link_down_work);
7692                 }
7693         }
7694 }
7695
7696 static const char * const fm_config_txt[] = {
7697 [0] =
7698         "BadHeadDist: Distance violation between two head flits",
7699 [1] =
7700         "BadTailDist: Distance violation between two tail flits",
7701 [2] =
7702         "BadCtrlDist: Distance violation between two credit control flits",
7703 [3] =
7704         "BadCrdAck: Credits return for unsupported VL",
7705 [4] =
7706         "UnsupportedVLMarker: Received VL Marker",
7707 [5] =
7708         "BadPreempt: Exceeded the preemption nesting level",
7709 [6] =
7710         "BadControlFlit: Received unsupported control flit",
7711 /* no 7 */
7712 [8] =
7713         "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7714 };
7715
7716 static const char * const port_rcv_txt[] = {
7717 [1] =
7718         "BadPktLen: Illegal PktLen",
7719 [2] =
7720         "PktLenTooLong: Packet longer than PktLen",
7721 [3] =
7722         "PktLenTooShort: Packet shorter than PktLen",
7723 [4] =
7724         "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7725 [5] =
7726         "BadDLID: Illegal DLID (0, doesn't match HFI)",
7727 [6] =
7728         "BadL2: Illegal L2 opcode",
7729 [7] =
7730         "BadSC: Unsupported SC",
7731 [9] =
7732         "BadRC: Illegal RC",
7733 [11] =
7734         "PreemptError: Preempting with same VL",
7735 [12] =
7736         "PreemptVL15: Preempting a VL15 packet",
7737 };
7738
7739 #define OPA_LDR_FMCONFIG_OFFSET 16
7740 #define OPA_LDR_PORTRCV_OFFSET 0
7741 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7742 {
7743         u64 info, hdr0, hdr1;
7744         const char *extra;
7745         char buf[96];
7746         struct hfi1_pportdata *ppd = dd->pport;
7747         u8 lcl_reason = 0;
7748         int do_bounce = 0;
7749
7750         if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7751                 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7752                         info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7753                         dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7754                         /* set status bit */
7755                         dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7756                 }
7757                 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7758         }
7759
7760         if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7761                 struct hfi1_pportdata *ppd = dd->pport;
7762                 /* this counter saturates at (2^32) - 1 */
7763                 if (ppd->link_downed < (u32)UINT_MAX)
7764                         ppd->link_downed++;
7765                 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7766         }
7767
7768         if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7769                 u8 reason_valid = 1;
7770
7771                 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7772                 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7773                         dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7774                         /* set status bit */
7775                         dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7776                 }
7777                 switch (info) {
7778                 case 0:
7779                 case 1:
7780                 case 2:
7781                 case 3:
7782                 case 4:
7783                 case 5:
7784                 case 6:
7785                         extra = fm_config_txt[info];
7786                         break;
7787                 case 8:
7788                         extra = fm_config_txt[info];
7789                         if (ppd->port_error_action &
7790                             OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7791                                 do_bounce = 1;
7792                                 /*
7793                                  * lcl_reason cannot be derived from info
7794                                  * for this error
7795                                  */
7796                                 lcl_reason =
7797                                   OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7798                         }
7799                         break;
7800                 default:
7801                         reason_valid = 0;
7802                         snprintf(buf, sizeof(buf), "reserved%lld", info);
7803                         extra = buf;
7804                         break;
7805                 }
7806
7807                 if (reason_valid && !do_bounce) {
7808                         do_bounce = ppd->port_error_action &
7809                                         (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7810                         lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7811                 }
7812
7813                 /* just report this */
7814                 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra);
7815                 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7816         }
7817
7818         if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
7819                 u8 reason_valid = 1;
7820
7821                 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
7822                 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
7823                 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
7824                 if (!(dd->err_info_rcvport.status_and_code &
7825                       OPA_EI_STATUS_SMASK)) {
7826                         dd->err_info_rcvport.status_and_code =
7827                                 info & OPA_EI_CODE_SMASK;
7828                         /* set status bit */
7829                         dd->err_info_rcvport.status_and_code |=
7830                                 OPA_EI_STATUS_SMASK;
7831                         /*
7832                          * save first 2 flits in the packet that caused
7833                          * the error
7834                          */
7835                         dd->err_info_rcvport.packet_flit1 = hdr0;
7836                         dd->err_info_rcvport.packet_flit2 = hdr1;
7837                 }
7838                 switch (info) {
7839                 case 1:
7840                 case 2:
7841                 case 3:
7842                 case 4:
7843                 case 5:
7844                 case 6:
7845                 case 7:
7846                 case 9:
7847                 case 11:
7848                 case 12:
7849                         extra = port_rcv_txt[info];
7850                         break;
7851                 default:
7852                         reason_valid = 0;
7853                         snprintf(buf, sizeof(buf), "reserved%lld", info);
7854                         extra = buf;
7855                         break;
7856                 }
7857
7858                 if (reason_valid && !do_bounce) {
7859                         do_bounce = ppd->port_error_action &
7860                                         (1 << (OPA_LDR_PORTRCV_OFFSET + info));
7861                         lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
7862                 }
7863
7864                 /* just report this */
7865                 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra);
7866                 dd_dev_info(dd, "           hdr0 0x%llx, hdr1 0x%llx\n",
7867                             hdr0, hdr1);
7868
7869                 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
7870         }
7871
7872         if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
7873                 /* informative only */
7874                 dd_dev_info(dd, "8051 access to LCB blocked\n");
7875                 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
7876         }
7877         if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
7878                 /* informative only */
7879                 dd_dev_info(dd, "host access to LCB blocked\n");
7880                 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
7881         }
7882
7883         /* report any remaining errors */
7884         if (reg)
7885                 dd_dev_info(dd, "DCC Error: %s\n",
7886                             dcc_err_string(buf, sizeof(buf), reg));
7887
7888         if (lcl_reason == 0)
7889                 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
7890
7891         if (do_bounce) {
7892                 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
7893                 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
7894                 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
7895         }
7896 }
7897
7898 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7899 {
7900         char buf[96];
7901
7902         dd_dev_info(dd, "LCB Error: %s\n",
7903                     lcb_err_string(buf, sizeof(buf), reg));
7904 }
7905
7906 /*
7907  * CCE block DC interrupt.  Source is < 8.
7908  */
7909 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
7910 {
7911         const struct err_reg_info *eri = &dc_errs[source];
7912
7913         if (eri->handler) {
7914                 interrupt_clear_down(dd, 0, eri);
7915         } else if (source == 3 /* dc_lbm_int */) {
7916                 /*
7917                  * This indicates that a parity error has occurred on the
7918                  * address/control lines presented to the LBM.  The error
7919                  * is a single pulse, there is no associated error flag,
7920                  * and it is non-maskable.  This is because if a parity
7921                  * error occurs on the request the request is dropped.
7922                  * This should never occur, but it is nice to know if it
7923                  * ever does.
7924                  */
7925                 dd_dev_err(dd, "Parity error in DC LBM block\n");
7926         } else {
7927                 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
7928         }
7929 }
7930
7931 /*
7932  * TX block send credit interrupt.  Source is < 160.
7933  */
7934 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
7935 {
7936         sc_group_release_update(dd, source);
7937 }
7938
7939 /*
7940  * TX block SDMA interrupt.  Source is < 48.
7941  *
7942  * SDMA interrupts are grouped by type:
7943  *
7944  *       0 -  N-1 = SDma
7945  *       N - 2N-1 = SDmaProgress
7946  *      2N - 3N-1 = SDmaIdle
7947  */
7948 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
7949 {
7950         /* what interrupt */
7951         unsigned int what  = source / TXE_NUM_SDMA_ENGINES;
7952         /* which engine */
7953         unsigned int which = source % TXE_NUM_SDMA_ENGINES;
7954
7955 #ifdef CONFIG_SDMA_VERBOSITY
7956         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
7957                    slashstrip(__FILE__), __LINE__, __func__);
7958         sdma_dumpstate(&dd->per_sdma[which]);
7959 #endif
7960
7961         if (likely(what < 3 && which < dd->num_sdma)) {
7962                 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
7963         } else {
7964                 /* should not happen */
7965                 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
7966         }
7967 }
7968
7969 /*
7970  * RX block receive available interrupt.  Source is < 160.
7971  */
7972 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
7973 {
7974         struct hfi1_ctxtdata *rcd;
7975         char *err_detail;
7976
7977         if (likely(source < dd->num_rcv_contexts)) {
7978                 rcd = dd->rcd[source];
7979                 if (rcd) {
7980                         if (source < dd->first_user_ctxt)
7981                                 rcd->do_interrupt(rcd, 0);
7982                         else
7983                                 handle_user_interrupt(rcd);
7984                         return; /* OK */
7985                 }
7986                 /* received an interrupt, but no rcd */
7987                 err_detail = "dataless";
7988         } else {
7989                 /* received an interrupt, but are not using that context */
7990                 err_detail = "out of range";
7991         }
7992         dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
7993                    err_detail, source);
7994 }
7995
7996 /*
7997  * RX block receive urgent interrupt.  Source is < 160.
7998  */
7999 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8000 {
8001         struct hfi1_ctxtdata *rcd;
8002         char *err_detail;
8003
8004         if (likely(source < dd->num_rcv_contexts)) {
8005                 rcd = dd->rcd[source];
8006                 if (rcd) {
8007                         /* only pay attention to user urgent interrupts */
8008                         if (source >= dd->first_user_ctxt)
8009                                 handle_user_interrupt(rcd);
8010                         return; /* OK */
8011                 }
8012                 /* received an interrupt, but no rcd */
8013                 err_detail = "dataless";
8014         } else {
8015                 /* received an interrupt, but are not using that context */
8016                 err_detail = "out of range";
8017         }
8018         dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
8019                    err_detail, source);
8020 }
8021
8022 /*
8023  * Reserved range interrupt.  Should not be called in normal operation.
8024  */
8025 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8026 {
8027         char name[64];
8028
8029         dd_dev_err(dd, "unexpected %s interrupt\n",
8030                    is_reserved_name(name, sizeof(name), source));
8031 }
8032
8033 static const struct is_table is_table[] = {
8034 /*
8035  * start                 end
8036  *                              name func               interrupt func
8037  */
8038 { IS_GENERAL_ERR_START,  IS_GENERAL_ERR_END,
8039                                 is_misc_err_name,       is_misc_err_int },
8040 { IS_SDMAENG_ERR_START,  IS_SDMAENG_ERR_END,
8041                                 is_sdma_eng_err_name,   is_sdma_eng_err_int },
8042 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8043                                 is_sendctxt_err_name,   is_sendctxt_err_int },
8044 { IS_SDMA_START,             IS_SDMA_END,
8045                                 is_sdma_eng_name,       is_sdma_eng_int },
8046 { IS_VARIOUS_START,          IS_VARIOUS_END,
8047                                 is_various_name,        is_various_int },
8048 { IS_DC_START,       IS_DC_END,
8049                                 is_dc_name,             is_dc_int },
8050 { IS_RCVAVAIL_START,     IS_RCVAVAIL_END,
8051                                 is_rcv_avail_name,      is_rcv_avail_int },
8052 { IS_RCVURGENT_START,    IS_RCVURGENT_END,
8053                                 is_rcv_urgent_name,     is_rcv_urgent_int },
8054 { IS_SENDCREDIT_START,   IS_SENDCREDIT_END,
8055                                 is_send_credit_name,    is_send_credit_int},
8056 { IS_RESERVED_START,     IS_RESERVED_END,
8057                                 is_reserved_name,       is_reserved_int},
8058 };
8059
8060 /*
8061  * Interrupt source interrupt - called when the given source has an interrupt.
8062  * Source is a bit index into an array of 64-bit integers.
8063  */
8064 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8065 {
8066         const struct is_table *entry;
8067
8068         /* avoids a double compare by walking the table in-order */
8069         for (entry = &is_table[0]; entry->is_name; entry++) {
8070                 if (source < entry->end) {
8071                         trace_hfi1_interrupt(dd, entry, source);
8072                         entry->is_int(dd, source - entry->start);
8073                         return;
8074                 }
8075         }
8076         /* fell off the end */
8077         dd_dev_err(dd, "invalid interrupt source %u\n", source);
8078 }
8079
8080 /*
8081  * General interrupt handler.  This is able to correctly handle
8082  * all interrupts in case INTx is used.
8083  */
8084 static irqreturn_t general_interrupt(int irq, void *data)
8085 {
8086         struct hfi1_devdata *dd = data;
8087         u64 regs[CCE_NUM_INT_CSRS];
8088         u32 bit;
8089         int i;
8090
8091         this_cpu_inc(*dd->int_counter);
8092
8093         /* phase 1: scan and clear all handled interrupts */
8094         for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8095                 if (dd->gi_mask[i] == 0) {
8096                         regs[i] = 0;    /* used later */
8097                         continue;
8098                 }
8099                 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8100                                 dd->gi_mask[i];
8101                 /* only clear if anything is set */
8102                 if (regs[i])
8103                         write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8104         }
8105
8106         /* phase 2: call the appropriate handler */
8107         for_each_set_bit(bit, (unsigned long *)&regs[0],
8108                          CCE_NUM_INT_CSRS * 64) {
8109                 is_interrupt(dd, bit);
8110         }
8111
8112         return IRQ_HANDLED;
8113 }
8114
8115 static irqreturn_t sdma_interrupt(int irq, void *data)
8116 {
8117         struct sdma_engine *sde = data;
8118         struct hfi1_devdata *dd = sde->dd;
8119         u64 status;
8120
8121 #ifdef CONFIG_SDMA_VERBOSITY
8122         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8123                    slashstrip(__FILE__), __LINE__, __func__);
8124         sdma_dumpstate(sde);
8125 #endif
8126
8127         this_cpu_inc(*dd->int_counter);
8128
8129         /* This read_csr is really bad in the hot path */
8130         status = read_csr(dd,
8131                           CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8132                           & sde->imask;
8133         if (likely(status)) {
8134                 /* clear the interrupt(s) */
8135                 write_csr(dd,
8136                           CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8137                           status);
8138
8139                 /* handle the interrupt(s) */
8140                 sdma_engine_interrupt(sde, status);
8141         } else
8142                 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
8143                            sde->this_idx);
8144
8145         return IRQ_HANDLED;
8146 }
8147
8148 /*
8149  * Clear the receive interrupt.  Use a read of the interrupt clear CSR
8150  * to insure that the write completed.  This does NOT guarantee that
8151  * queued DMA writes to memory from the chip are pushed.
8152  */
8153 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8154 {
8155         struct hfi1_devdata *dd = rcd->dd;
8156         u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8157
8158         mmiowb();       /* make sure everything before is written */
8159         write_csr(dd, addr, rcd->imask);
8160         /* force the above write on the chip and get a value back */
8161         (void)read_csr(dd, addr);
8162 }
8163
8164 /* force the receive interrupt */
8165 void force_recv_intr(struct hfi1_ctxtdata *rcd)
8166 {
8167         write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8168 }
8169
8170 /*
8171  * Return non-zero if a packet is present.
8172  *
8173  * This routine is called when rechecking for packets after the RcvAvail
8174  * interrupt has been cleared down.  First, do a quick check of memory for
8175  * a packet present.  If not found, use an expensive CSR read of the context
8176  * tail to determine the actual tail.  The CSR read is necessary because there
8177  * is no method to push pending DMAs to memory other than an interrupt and we
8178  * are trying to determine if we need to force an interrupt.
8179  */
8180 static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8181 {
8182         u32 tail;
8183         int present;
8184
8185         if (!HFI1_CAP_IS_KSET(DMA_RTAIL))
8186                 present = (rcd->seq_cnt ==
8187                                 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
8188         else /* is RDMA rtail */
8189                 present = (rcd->head != get_rcvhdrtail(rcd));
8190
8191         if (present)
8192                 return 1;
8193
8194         /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8195         tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8196         return rcd->head != tail;
8197 }
8198
8199 /*
8200  * Receive packet IRQ handler.  This routine expects to be on its own IRQ.
8201  * This routine will try to handle packets immediately (latency), but if
8202  * it finds too many, it will invoke the thread handler (bandwitdh).  The
8203  * chip receive interrupt is *not* cleared down until this or the thread (if
8204  * invoked) is finished.  The intent is to avoid extra interrupts while we
8205  * are processing packets anyway.
8206  */
8207 static irqreturn_t receive_context_interrupt(int irq, void *data)
8208 {
8209         struct hfi1_ctxtdata *rcd = data;
8210         struct hfi1_devdata *dd = rcd->dd;
8211         int disposition;
8212         int present;
8213
8214         trace_hfi1_receive_interrupt(dd, rcd->ctxt);
8215         this_cpu_inc(*dd->int_counter);
8216         aspm_ctx_disable(rcd);
8217
8218         /* receive interrupt remains blocked while processing packets */
8219         disposition = rcd->do_interrupt(rcd, 0);
8220
8221         /*
8222          * Too many packets were seen while processing packets in this
8223          * IRQ handler.  Invoke the handler thread.  The receive interrupt
8224          * remains blocked.
8225          */
8226         if (disposition == RCV_PKT_LIMIT)
8227                 return IRQ_WAKE_THREAD;
8228
8229         /*
8230          * The packet processor detected no more packets.  Clear the receive
8231          * interrupt and recheck for a packet packet that may have arrived
8232          * after the previous check and interrupt clear.  If a packet arrived,
8233          * force another interrupt.
8234          */
8235         clear_recv_intr(rcd);
8236         present = check_packet_present(rcd);
8237         if (present)
8238                 force_recv_intr(rcd);
8239
8240         return IRQ_HANDLED;
8241 }
8242
8243 /*
8244  * Receive packet thread handler.  This expects to be invoked with the
8245  * receive interrupt still blocked.
8246  */
8247 static irqreturn_t receive_context_thread(int irq, void *data)
8248 {
8249         struct hfi1_ctxtdata *rcd = data;
8250         int present;
8251
8252         /* receive interrupt is still blocked from the IRQ handler */
8253         (void)rcd->do_interrupt(rcd, 1);
8254
8255         /*
8256          * The packet processor will only return if it detected no more
8257          * packets.  Hold IRQs here so we can safely clear the interrupt and
8258          * recheck for a packet that may have arrived after the previous
8259          * check and the interrupt clear.  If a packet arrived, force another
8260          * interrupt.
8261          */
8262         local_irq_disable();
8263         clear_recv_intr(rcd);
8264         present = check_packet_present(rcd);
8265         if (present)
8266                 force_recv_intr(rcd);
8267         local_irq_enable();
8268
8269         return IRQ_HANDLED;
8270 }
8271
8272 /* ========================================================================= */
8273
8274 u32 read_physical_state(struct hfi1_devdata *dd)
8275 {
8276         u64 reg;
8277
8278         reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8279         return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8280                                 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8281 }
8282
8283 u32 read_logical_state(struct hfi1_devdata *dd)
8284 {
8285         u64 reg;
8286
8287         reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8288         return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8289                                 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8290 }
8291
8292 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8293 {
8294         u64 reg;
8295
8296         reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8297         /* clear current state, set new state */
8298         reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8299         reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8300         write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8301 }
8302
8303 /*
8304  * Use the 8051 to read a LCB CSR.
8305  */
8306 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8307 {
8308         u32 regno;
8309         int ret;
8310
8311         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8312                 if (acquire_lcb_access(dd, 0) == 0) {
8313                         *data = read_csr(dd, addr);
8314                         release_lcb_access(dd, 0);
8315                         return 0;
8316                 }
8317                 return -EBUSY;
8318         }
8319
8320         /* register is an index of LCB registers: (offset - base) / 8 */
8321         regno = (addr - DC_LCB_CFG_RUN) >> 3;
8322         ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8323         if (ret != HCMD_SUCCESS)
8324                 return -EBUSY;
8325         return 0;
8326 }
8327
8328 /*
8329  * Read an LCB CSR.  Access may not be in host control, so check.
8330  * Return 0 on success, -EBUSY on failure.
8331  */
8332 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8333 {
8334         struct hfi1_pportdata *ppd = dd->pport;
8335
8336         /* if up, go through the 8051 for the value */
8337         if (ppd->host_link_state & HLS_UP)
8338                 return read_lcb_via_8051(dd, addr, data);
8339         /* if going up or down, no access */
8340         if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8341                 return -EBUSY;
8342         /* otherwise, host has access */
8343         *data = read_csr(dd, addr);
8344         return 0;
8345 }
8346
8347 /*
8348  * Use the 8051 to write a LCB CSR.
8349  */
8350 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8351 {
8352         u32 regno;
8353         int ret;
8354
8355         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8356             (dd->dc8051_ver < dc8051_ver(0, 20))) {
8357                 if (acquire_lcb_access(dd, 0) == 0) {
8358                         write_csr(dd, addr, data);
8359                         release_lcb_access(dd, 0);
8360                         return 0;
8361                 }
8362                 return -EBUSY;
8363         }
8364
8365         /* register is an index of LCB registers: (offset - base) / 8 */
8366         regno = (addr - DC_LCB_CFG_RUN) >> 3;
8367         ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8368         if (ret != HCMD_SUCCESS)
8369                 return -EBUSY;
8370         return 0;
8371 }
8372
8373 /*
8374  * Write an LCB CSR.  Access may not be in host control, so check.
8375  * Return 0 on success, -EBUSY on failure.
8376  */
8377 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8378 {
8379         struct hfi1_pportdata *ppd = dd->pport;
8380
8381         /* if up, go through the 8051 for the value */
8382         if (ppd->host_link_state & HLS_UP)
8383                 return write_lcb_via_8051(dd, addr, data);
8384         /* if going up or down, no access */
8385         if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8386                 return -EBUSY;
8387         /* otherwise, host has access */
8388         write_csr(dd, addr, data);
8389         return 0;
8390 }
8391
8392 /*
8393  * Returns:
8394  *      < 0 = Linux error, not able to get access
8395  *      > 0 = 8051 command RETURN_CODE
8396  */
8397 static int do_8051_command(
8398         struct hfi1_devdata *dd,
8399         u32 type,
8400         u64 in_data,
8401         u64 *out_data)
8402 {
8403         u64 reg, completed;
8404         int return_code;
8405         unsigned long flags;
8406         unsigned long timeout;
8407
8408         hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8409
8410         /*
8411          * Alternative to holding the lock for a long time:
8412          * - keep busy wait - have other users bounce off
8413          */
8414         spin_lock_irqsave(&dd->dc8051_lock, flags);
8415
8416         /* We can't send any commands to the 8051 if it's in reset */
8417         if (dd->dc_shutdown) {
8418                 return_code = -ENODEV;
8419                 goto fail;
8420         }
8421
8422         /*
8423          * If an 8051 host command timed out previously, then the 8051 is
8424          * stuck.
8425          *
8426          * On first timeout, attempt to reset and restart the entire DC
8427          * block (including 8051). (Is this too big of a hammer?)
8428          *
8429          * If the 8051 times out a second time, the reset did not bring it
8430          * back to healthy life. In that case, fail any subsequent commands.
8431          */
8432         if (dd->dc8051_timed_out) {
8433                 if (dd->dc8051_timed_out > 1) {
8434                         dd_dev_err(dd,
8435                                    "Previous 8051 host command timed out, skipping command %u\n",
8436                                    type);
8437                         return_code = -ENXIO;
8438                         goto fail;
8439                 }
8440                 spin_unlock_irqrestore(&dd->dc8051_lock, flags);
8441                 dc_shutdown(dd);
8442                 dc_start(dd);
8443                 spin_lock_irqsave(&dd->dc8051_lock, flags);
8444         }
8445
8446         /*
8447          * If there is no timeout, then the 8051 command interface is
8448          * waiting for a command.
8449          */
8450
8451         /*
8452          * When writing a LCB CSR, out_data contains the full value to
8453          * to be written, while in_data contains the relative LCB
8454          * address in 7:0.  Do the work here, rather than the caller,
8455          * of distrubting the write data to where it needs to go:
8456          *
8457          * Write data
8458          *   39:00 -> in_data[47:8]
8459          *   47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8460          *   63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8461          */
8462         if (type == HCMD_WRITE_LCB_CSR) {
8463                 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8464                 reg = ((((*out_data) >> 40) & 0xff) <<
8465                                 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8466                       | ((((*out_data) >> 48) & 0xffff) <<
8467                                 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8468                 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8469         }
8470
8471         /*
8472          * Do two writes: the first to stabilize the type and req_data, the
8473          * second to activate.
8474          */
8475         reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8476                         << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8477                 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8478                         << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8479         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8480         reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8481         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8482
8483         /* wait for completion, alternate: interrupt */
8484         timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8485         while (1) {
8486                 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8487                 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8488                 if (completed)
8489                         break;
8490                 if (time_after(jiffies, timeout)) {
8491                         dd->dc8051_timed_out++;
8492                         dd_dev_err(dd, "8051 host command %u timeout\n", type);
8493                         if (out_data)
8494                                 *out_data = 0;
8495                         return_code = -ETIMEDOUT;
8496                         goto fail;
8497                 }
8498                 udelay(2);
8499         }
8500
8501         if (out_data) {
8502                 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8503                                 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8504                 if (type == HCMD_READ_LCB_CSR) {
8505                         /* top 16 bits are in a different register */
8506                         *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8507                                 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8508                                 << (48
8509                                     - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8510                 }
8511         }
8512         return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8513                                 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8514         dd->dc8051_timed_out = 0;
8515         /*
8516          * Clear command for next user.
8517          */
8518         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8519
8520 fail:
8521         spin_unlock_irqrestore(&dd->dc8051_lock, flags);
8522
8523         return return_code;
8524 }
8525
8526 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8527 {
8528         return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8529 }
8530
8531 int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8532                      u8 lane_id, u32 config_data)
8533 {
8534         u64 data;
8535         int ret;
8536
8537         data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8538                 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8539                 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8540         ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8541         if (ret != HCMD_SUCCESS) {
8542                 dd_dev_err(dd,
8543                            "load 8051 config: field id %d, lane %d, err %d\n",
8544                            (int)field_id, (int)lane_id, ret);
8545         }
8546         return ret;
8547 }
8548
8549 /*
8550  * Read the 8051 firmware "registers".  Use the RAM directly.  Always
8551  * set the result, even on error.
8552  * Return 0 on success, -errno on failure
8553  */
8554 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8555                      u32 *result)
8556 {
8557         u64 big_data;
8558         u32 addr;
8559         int ret;
8560
8561         /* address start depends on the lane_id */
8562         if (lane_id < 4)
8563                 addr = (4 * NUM_GENERAL_FIELDS)
8564                         + (lane_id * 4 * NUM_LANE_FIELDS);
8565         else
8566                 addr = 0;
8567         addr += field_id * 4;
8568
8569         /* read is in 8-byte chunks, hardware will truncate the address down */
8570         ret = read_8051_data(dd, addr, 8, &big_data);
8571
8572         if (ret == 0) {
8573                 /* extract the 4 bytes we want */
8574                 if (addr & 0x4)
8575                         *result = (u32)(big_data >> 32);
8576                 else
8577                         *result = (u32)big_data;
8578         } else {
8579                 *result = 0;
8580                 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
8581                            __func__, lane_id, field_id);
8582         }
8583
8584         return ret;
8585 }
8586
8587 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8588                               u8 continuous)
8589 {
8590         u32 frame;
8591
8592         frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8593                 | power_management << POWER_MANAGEMENT_SHIFT;
8594         return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8595                                 GENERAL_CONFIG, frame);
8596 }
8597
8598 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8599                                  u16 vl15buf, u8 crc_sizes)
8600 {
8601         u32 frame;
8602
8603         frame = (u32)vau << VAU_SHIFT
8604                 | (u32)z << Z_SHIFT
8605                 | (u32)vcu << VCU_SHIFT
8606                 | (u32)vl15buf << VL15BUF_SHIFT
8607                 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8608         return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8609                                 GENERAL_CONFIG, frame);
8610 }
8611
8612 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits,
8613                                      u8 *flag_bits, u16 *link_widths)
8614 {
8615         u32 frame;
8616
8617         read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
8618                          &frame);
8619         *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8620         *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8621         *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8622 }
8623
8624 static int write_vc_local_link_width(struct hfi1_devdata *dd,
8625                                      u8 misc_bits,
8626                                      u8 flag_bits,
8627                                      u16 link_widths)
8628 {
8629         u32 frame;
8630
8631         frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8632                 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8633                 | (u32)link_widths << LINK_WIDTH_SHIFT;
8634         return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG,
8635                      frame);
8636 }
8637
8638 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8639                                  u8 device_rev)
8640 {
8641         u32 frame;
8642
8643         frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8644                 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8645         return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8646 }
8647
8648 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8649                                   u8 *device_rev)
8650 {
8651         u32 frame;
8652
8653         read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8654         *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8655         *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8656                         & REMOTE_DEVICE_REV_MASK;
8657 }
8658
8659 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b)
8660 {
8661         u32 frame;
8662
8663         read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8664         *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK;
8665         *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK;
8666 }
8667
8668 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8669                                u8 *continuous)
8670 {
8671         u32 frame;
8672
8673         read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8674         *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8675                                         & POWER_MANAGEMENT_MASK;
8676         *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
8677                                         & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
8678 }
8679
8680 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
8681                                   u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
8682 {
8683         u32 frame;
8684
8685         read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
8686         *vau = (frame >> VAU_SHIFT) & VAU_MASK;
8687         *z = (frame >> Z_SHIFT) & Z_MASK;
8688         *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
8689         *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
8690         *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
8691 }
8692
8693 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
8694                                       u8 *remote_tx_rate,
8695                                       u16 *link_widths)
8696 {
8697         u32 frame;
8698
8699         read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
8700                          &frame);
8701         *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
8702                                 & REMOTE_TX_RATE_MASK;
8703         *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8704 }
8705
8706 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
8707 {
8708         u32 frame;
8709
8710         read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
8711         *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
8712 }
8713
8714 static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed)
8715 {
8716         u32 frame;
8717
8718         read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame);
8719         *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK;
8720 }
8721
8722 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
8723 {
8724         read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
8725 }
8726
8727 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
8728 {
8729         read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
8730 }
8731
8732 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
8733 {
8734         u32 frame;
8735         int ret;
8736
8737         *link_quality = 0;
8738         if (dd->pport->host_link_state & HLS_UP) {
8739                 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
8740                                        &frame);
8741                 if (ret == 0)
8742                         *link_quality = (frame >> LINK_QUALITY_SHIFT)
8743                                                 & LINK_QUALITY_MASK;
8744         }
8745 }
8746
8747 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
8748 {
8749         u32 frame;
8750
8751         read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
8752         *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
8753 }
8754
8755 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
8756 {
8757         u32 frame;
8758
8759         read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
8760         *ldr = (frame & 0xff);
8761 }
8762
8763 static int read_tx_settings(struct hfi1_devdata *dd,
8764                             u8 *enable_lane_tx,
8765                             u8 *tx_polarity_inversion,
8766                             u8 *rx_polarity_inversion,
8767                             u8 *max_rate)
8768 {
8769         u32 frame;
8770         int ret;
8771
8772         ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
8773         *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
8774                                 & ENABLE_LANE_TX_MASK;
8775         *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
8776                                 & TX_POLARITY_INVERSION_MASK;
8777         *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
8778                                 & RX_POLARITY_INVERSION_MASK;
8779         *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
8780         return ret;
8781 }
8782
8783 static int write_tx_settings(struct hfi1_devdata *dd,
8784                              u8 enable_lane_tx,
8785                              u8 tx_polarity_inversion,
8786                              u8 rx_polarity_inversion,
8787                              u8 max_rate)
8788 {
8789         u32 frame;
8790
8791         /* no need to mask, all variable sizes match field widths */
8792         frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
8793                 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
8794                 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
8795                 | max_rate << MAX_RATE_SHIFT;
8796         return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
8797 }
8798
8799 static void check_fabric_firmware_versions(struct hfi1_devdata *dd)
8800 {
8801         u32 frame, version, prod_id;
8802         int ret, lane;
8803
8804         /* 4 lanes */
8805         for (lane = 0; lane < 4; lane++) {
8806                 ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame);
8807                 if (ret) {
8808                         dd_dev_err(dd,
8809                                    "Unable to read lane %d firmware details\n",
8810                                    lane);
8811                         continue;
8812                 }
8813                 version = (frame >> SPICO_ROM_VERSION_SHIFT)
8814                                         & SPICO_ROM_VERSION_MASK;
8815                 prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT)
8816                                         & SPICO_ROM_PROD_ID_MASK;
8817                 dd_dev_info(dd,
8818                             "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n",
8819                             lane, version, prod_id);
8820         }
8821 }
8822
8823 /*
8824  * Read an idle LCB message.
8825  *
8826  * Returns 0 on success, -EINVAL on error
8827  */
8828 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
8829 {
8830         int ret;
8831
8832         ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
8833         if (ret != HCMD_SUCCESS) {
8834                 dd_dev_err(dd, "read idle message: type %d, err %d\n",
8835                            (u32)type, ret);
8836                 return -EINVAL;
8837         }
8838         dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
8839         /* return only the payload as we already know the type */
8840         *data_out >>= IDLE_PAYLOAD_SHIFT;
8841         return 0;
8842 }
8843
8844 /*
8845  * Read an idle SMA message.  To be done in response to a notification from
8846  * the 8051.
8847  *
8848  * Returns 0 on success, -EINVAL on error
8849  */
8850 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
8851 {
8852         return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
8853                                  data);
8854 }
8855
8856 /*
8857  * Send an idle LCB message.
8858  *
8859  * Returns 0 on success, -EINVAL on error
8860  */
8861 static int send_idle_message(struct hfi1_devdata *dd, u64 data)
8862 {
8863         int ret;
8864
8865         dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
8866         ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
8867         if (ret != HCMD_SUCCESS) {
8868                 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
8869                            data, ret);
8870                 return -EINVAL;
8871         }
8872         return 0;
8873 }
8874
8875 /*
8876  * Send an idle SMA message.
8877  *
8878  * Returns 0 on success, -EINVAL on error
8879  */
8880 int send_idle_sma(struct hfi1_devdata *dd, u64 message)
8881 {
8882         u64 data;
8883
8884         data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
8885                 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
8886         return send_idle_message(dd, data);
8887 }
8888
8889 /*
8890  * Initialize the LCB then do a quick link up.  This may or may not be
8891  * in loopback.
8892  *
8893  * return 0 on success, -errno on error
8894  */
8895 static int do_quick_linkup(struct hfi1_devdata *dd)
8896 {
8897         u64 reg;
8898         unsigned long timeout;
8899         int ret;
8900
8901         lcb_shutdown(dd, 0);
8902
8903         if (loopback) {
8904                 /* LCB_CFG_LOOPBACK.VAL = 2 */
8905                 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
8906                 write_csr(dd, DC_LCB_CFG_LOOPBACK,
8907                           IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
8908                 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
8909         }
8910
8911         /* start the LCBs */
8912         /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
8913         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
8914
8915         /* simulator only loopback steps */
8916         if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8917                 /* LCB_CFG_RUN.EN = 1 */
8918                 write_csr(dd, DC_LCB_CFG_RUN,
8919                           1ull << DC_LCB_CFG_RUN_EN_SHIFT);
8920
8921                 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
8922                 timeout = jiffies + msecs_to_jiffies(10);
8923                 while (1) {
8924                         reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
8925                         if (reg)
8926                                 break;
8927                         if (time_after(jiffies, timeout)) {
8928                                 dd_dev_err(dd,
8929                                            "timeout waiting for LINK_TRANSFER_ACTIVE\n");
8930                                 return -ETIMEDOUT;
8931                         }
8932                         udelay(2);
8933                 }
8934
8935                 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
8936                           1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
8937         }
8938
8939         if (!loopback) {
8940                 /*
8941                  * When doing quick linkup and not in loopback, both
8942                  * sides must be done with LCB set-up before either
8943                  * starts the quick linkup.  Put a delay here so that
8944                  * both sides can be started and have a chance to be
8945                  * done with LCB set up before resuming.
8946                  */
8947                 dd_dev_err(dd,
8948                            "Pausing for peer to be finished with LCB set up\n");
8949                 msleep(5000);
8950                 dd_dev_err(dd, "Continuing with quick linkup\n");
8951         }
8952
8953         write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
8954         set_8051_lcb_access(dd);
8955
8956         /*
8957          * State "quick" LinkUp request sets the physical link state to
8958          * LinkUp without a verify capability sequence.
8959          * This state is in simulator v37 and later.
8960          */
8961         ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
8962         if (ret != HCMD_SUCCESS) {
8963                 dd_dev_err(dd,
8964                            "%s: set physical link state to quick LinkUp failed with return %d\n",
8965                            __func__, ret);
8966
8967                 set_host_lcb_access(dd);
8968                 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
8969
8970                 if (ret >= 0)
8971                         ret = -EINVAL;
8972                 return ret;
8973         }
8974
8975         return 0; /* success */
8976 }
8977
8978 /*
8979  * Set the SerDes to internal loopback mode.
8980  * Returns 0 on success, -errno on error.
8981  */
8982 static int set_serdes_loopback_mode(struct hfi1_devdata *dd)
8983 {
8984         int ret;
8985
8986         ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK);
8987         if (ret == HCMD_SUCCESS)
8988                 return 0;
8989         dd_dev_err(dd,
8990                    "Set physical link state to SerDes Loopback failed with return %d\n",
8991                    ret);
8992         if (ret >= 0)
8993                 ret = -EINVAL;
8994         return ret;
8995 }
8996
8997 /*
8998  * Do all special steps to set up loopback.
8999  */
9000 static int init_loopback(struct hfi1_devdata *dd)
9001 {
9002         dd_dev_info(dd, "Entering loopback mode\n");
9003
9004         /* all loopbacks should disable self GUID check */
9005         write_csr(dd, DC_DC8051_CFG_MODE,
9006                   (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
9007
9008         /*
9009          * The simulator has only one loopback option - LCB.  Switch
9010          * to that option, which includes quick link up.
9011          *
9012          * Accept all valid loopback values.
9013          */
9014         if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9015             (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9016              loopback == LOOPBACK_CABLE)) {
9017                 loopback = LOOPBACK_LCB;
9018                 quick_linkup = 1;
9019                 return 0;
9020         }
9021
9022         /* handle serdes loopback */
9023         if (loopback == LOOPBACK_SERDES) {
9024                 /* internal serdes loopack needs quick linkup on RTL */
9025                 if (dd->icode == ICODE_RTL_SILICON)
9026                         quick_linkup = 1;
9027                 return set_serdes_loopback_mode(dd);
9028         }
9029
9030         /* LCB loopback - handled at poll time */
9031         if (loopback == LOOPBACK_LCB) {
9032                 quick_linkup = 1; /* LCB is always quick linkup */
9033
9034                 /* not supported in emulation due to emulation RTL changes */
9035                 if (dd->icode == ICODE_FPGA_EMULATION) {
9036                         dd_dev_err(dd,
9037                                    "LCB loopback not supported in emulation\n");
9038                         return -EINVAL;
9039                 }
9040                 return 0;
9041         }
9042
9043         /* external cable loopback requires no extra steps */
9044         if (loopback == LOOPBACK_CABLE)
9045                 return 0;
9046
9047         dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9048         return -EINVAL;
9049 }
9050
9051 /*
9052  * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
9053  * used in the Verify Capability link width attribute.
9054  */
9055 static u16 opa_to_vc_link_widths(u16 opa_widths)
9056 {
9057         int i;
9058         u16 result = 0;
9059
9060         static const struct link_bits {
9061                 u16 from;
9062                 u16 to;
9063         } opa_link_xlate[] = {
9064                 { OPA_LINK_WIDTH_1X, 1 << (1 - 1)  },
9065                 { OPA_LINK_WIDTH_2X, 1 << (2 - 1)  },
9066                 { OPA_LINK_WIDTH_3X, 1 << (3 - 1)  },
9067                 { OPA_LINK_WIDTH_4X, 1 << (4 - 1)  },
9068         };
9069
9070         for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9071                 if (opa_widths & opa_link_xlate[i].from)
9072                         result |= opa_link_xlate[i].to;
9073         }
9074         return result;
9075 }
9076
9077 /*
9078  * Set link attributes before moving to polling.
9079  */
9080 static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9081 {
9082         struct hfi1_devdata *dd = ppd->dd;
9083         u8 enable_lane_tx;
9084         u8 tx_polarity_inversion;
9085         u8 rx_polarity_inversion;
9086         int ret;
9087
9088         /* reset our fabric serdes to clear any lingering problems */
9089         fabric_serdes_reset(dd);
9090
9091         /* set the local tx rate - need to read-modify-write */
9092         ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
9093                                &rx_polarity_inversion, &ppd->local_tx_rate);
9094         if (ret)
9095                 goto set_local_link_attributes_fail;
9096
9097         if (dd->dc8051_ver < dc8051_ver(0, 20)) {
9098                 /* set the tx rate to the fastest enabled */
9099                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9100                         ppd->local_tx_rate = 1;
9101                 else
9102                         ppd->local_tx_rate = 0;
9103         } else {
9104                 /* set the tx rate to all enabled */
9105                 ppd->local_tx_rate = 0;
9106                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9107                         ppd->local_tx_rate |= 2;
9108                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9109                         ppd->local_tx_rate |= 1;
9110         }
9111
9112         enable_lane_tx = 0xF; /* enable all four lanes */
9113         ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
9114                                 rx_polarity_inversion, ppd->local_tx_rate);
9115         if (ret != HCMD_SUCCESS)
9116                 goto set_local_link_attributes_fail;
9117
9118         /*
9119          * DC supports continuous updates.
9120          */
9121         ret = write_vc_local_phy(dd,
9122                                  0 /* no power management */,
9123                                  1 /* continuous updates */);
9124         if (ret != HCMD_SUCCESS)
9125                 goto set_local_link_attributes_fail;
9126
9127         /* z=1 in the next call: AU of 0 is not supported by the hardware */
9128         ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9129                                     ppd->port_crc_mode_enabled);
9130         if (ret != HCMD_SUCCESS)
9131                 goto set_local_link_attributes_fail;
9132
9133         ret = write_vc_local_link_width(dd, 0, 0,
9134                                         opa_to_vc_link_widths(
9135                                                 ppd->link_width_enabled));
9136         if (ret != HCMD_SUCCESS)
9137                 goto set_local_link_attributes_fail;
9138
9139         /* let peer know who we are */
9140         ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9141         if (ret == HCMD_SUCCESS)
9142                 return 0;
9143
9144 set_local_link_attributes_fail:
9145         dd_dev_err(dd,
9146                    "Failed to set local link attributes, return 0x%x\n",
9147                    ret);
9148         return ret;
9149 }
9150
9151 /*
9152  * Call this to start the link.
9153  * Do not do anything if the link is disabled.
9154  * Returns 0 if link is disabled, moved to polling, or the driver is not ready.
9155  */
9156 int start_link(struct hfi1_pportdata *ppd)
9157 {
9158         if (!ppd->link_enabled) {
9159                 dd_dev_info(ppd->dd,
9160                             "%s: stopping link start because link is disabled\n",
9161                             __func__);
9162                 return 0;
9163         }
9164         if (!ppd->driver_link_ready) {
9165                 dd_dev_info(ppd->dd,
9166                             "%s: stopping link start because driver is not ready\n",
9167                             __func__);
9168                 return 0;
9169         }
9170
9171         return set_link_state(ppd, HLS_DN_POLL);
9172 }
9173
9174 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9175 {
9176         struct hfi1_devdata *dd = ppd->dd;
9177         u64 mask;
9178         unsigned long timeout;
9179
9180         /*
9181          * Check for QSFP interrupt for t_init (SFF 8679)
9182          */
9183         timeout = jiffies + msecs_to_jiffies(2000);
9184         while (1) {
9185                 mask = read_csr(dd, dd->hfi1_id ?
9186                                 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9187                 if (!(mask & QSFP_HFI0_INT_N)) {
9188                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR :
9189                                   ASIC_QSFP1_CLEAR, QSFP_HFI0_INT_N);
9190                         break;
9191                 }
9192                 if (time_after(jiffies, timeout)) {
9193                         dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9194                                     __func__);
9195                         break;
9196                 }
9197                 udelay(2);
9198         }
9199 }
9200
9201 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9202 {
9203         struct hfi1_devdata *dd = ppd->dd;
9204         u64 mask;
9205
9206         mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9207         if (enable)
9208                 mask |= (u64)QSFP_HFI0_INT_N;
9209         else
9210                 mask &= ~(u64)QSFP_HFI0_INT_N;
9211         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9212 }
9213
9214 void reset_qsfp(struct hfi1_pportdata *ppd)
9215 {
9216         struct hfi1_devdata *dd = ppd->dd;
9217         u64 mask, qsfp_mask;
9218
9219         /* Disable INT_N from triggering QSFP interrupts */
9220         set_qsfp_int_n(ppd, 0);
9221
9222         /* Reset the QSFP */
9223         mask = (u64)QSFP_HFI0_RESET_N;
9224
9225         qsfp_mask = read_csr(dd,
9226                              dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
9227         qsfp_mask &= ~mask;
9228         write_csr(dd,
9229                   dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9230
9231         udelay(10);
9232
9233         qsfp_mask |= mask;
9234         write_csr(dd,
9235                   dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9236
9237         wait_for_qsfp_init(ppd);
9238
9239         /*
9240          * Allow INT_N to trigger the QSFP interrupt to watch
9241          * for alarms and warnings
9242          */
9243         set_qsfp_int_n(ppd, 1);
9244 }
9245
9246 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9247                                         u8 *qsfp_interrupt_status)
9248 {
9249         struct hfi1_devdata *dd = ppd->dd;
9250
9251         if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
9252             (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9253                 dd_dev_info(dd, "%s: QSFP cable on fire\n",
9254                             __func__);
9255
9256         if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
9257             (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9258                 dd_dev_info(dd, "%s: QSFP cable temperature too low\n",
9259                             __func__);
9260
9261         /*
9262          * The remaining alarms/warnings don't matter if the link is down.
9263          */
9264         if (ppd->host_link_state & HLS_DOWN)
9265                 return 0;
9266
9267         if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
9268             (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9269                 dd_dev_info(dd, "%s: QSFP supply voltage too high\n",
9270                             __func__);
9271
9272         if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
9273             (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9274                 dd_dev_info(dd, "%s: QSFP supply voltage too low\n",
9275                             __func__);
9276
9277         /* Byte 2 is vendor specific */
9278
9279         if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
9280             (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9281                 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too high\n",
9282                             __func__);
9283
9284         if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
9285             (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9286                 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too low\n",
9287                             __func__);
9288
9289         if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
9290             (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9291                 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too high\n",
9292                             __func__);
9293
9294         if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
9295             (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9296                 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too low\n",
9297                             __func__);
9298
9299         if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
9300             (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9301                 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too high\n",
9302                             __func__);
9303
9304         if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
9305             (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9306                 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too low\n",
9307                             __func__);
9308
9309         if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
9310             (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9311                 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too high\n",
9312                             __func__);
9313
9314         if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
9315             (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9316                 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too low\n",
9317                             __func__);
9318
9319         if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
9320             (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9321                 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too high\n",
9322                             __func__);
9323
9324         if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
9325             (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9326                 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too low\n",
9327                             __func__);
9328
9329         if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
9330             (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9331                 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too high\n",
9332                             __func__);
9333
9334         if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
9335             (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9336                 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too low\n",
9337                             __func__);
9338
9339         /* Bytes 9-10 and 11-12 are reserved */
9340         /* Bytes 13-15 are vendor specific */
9341
9342         return 0;
9343 }
9344
9345 /* This routine will only be scheduled if the QSFP module present is asserted */
9346 void qsfp_event(struct work_struct *work)
9347 {
9348         struct qsfp_data *qd;
9349         struct hfi1_pportdata *ppd;
9350         struct hfi1_devdata *dd;
9351
9352         qd = container_of(work, struct qsfp_data, qsfp_work);
9353         ppd = qd->ppd;
9354         dd = ppd->dd;
9355
9356         /* Sanity check */
9357         if (!qsfp_mod_present(ppd))
9358                 return;
9359
9360         /*
9361          * Turn DC back on after cable has been re-inserted. Up until
9362          * now, the DC has been in reset to save power.
9363          */
9364         dc_start(dd);
9365
9366         if (qd->cache_refresh_required) {
9367                 set_qsfp_int_n(ppd, 0);
9368
9369                 wait_for_qsfp_init(ppd);
9370
9371                 /*
9372                  * Allow INT_N to trigger the QSFP interrupt to watch
9373                  * for alarms and warnings
9374                  */
9375                 set_qsfp_int_n(ppd, 1);
9376
9377                 tune_serdes(ppd);
9378
9379                 start_link(ppd);
9380         }
9381
9382         if (qd->check_interrupt_flags) {
9383                 u8 qsfp_interrupt_status[16] = {0,};
9384
9385                 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9386                                   &qsfp_interrupt_status[0], 16) != 16) {
9387                         dd_dev_info(dd,
9388                                     "%s: Failed to read status of QSFP module\n",
9389                                     __func__);
9390                 } else {
9391                         unsigned long flags;
9392
9393                         handle_qsfp_error_conditions(
9394                                         ppd, qsfp_interrupt_status);
9395                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9396                         ppd->qsfp_info.check_interrupt_flags = 0;
9397                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
9398                                                flags);
9399                 }
9400         }
9401 }
9402
9403 static void init_qsfp_int(struct hfi1_devdata *dd)
9404 {
9405         struct hfi1_pportdata *ppd = dd->pport;
9406         u64 qsfp_mask, cce_int_mask;
9407         const int qsfp1_int_smask = QSFP1_INT % 64;
9408         const int qsfp2_int_smask = QSFP2_INT % 64;
9409
9410         /*
9411          * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0
9412          * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR,
9413          * therefore just one of QSFP1_INT/QSFP2_INT can be used to find
9414          * the index of the appropriate CSR in the CCEIntMask CSR array
9415          */
9416         cce_int_mask = read_csr(dd, CCE_INT_MASK +
9417                                 (8 * (QSFP1_INT / 64)));
9418         if (dd->hfi1_id) {
9419                 cce_int_mask &= ~((u64)1 << qsfp1_int_smask);
9420                 write_csr(dd, CCE_INT_MASK + (8 * (QSFP1_INT / 64)),
9421                           cce_int_mask);
9422         } else {
9423                 cce_int_mask &= ~((u64)1 << qsfp2_int_smask);
9424                 write_csr(dd, CCE_INT_MASK + (8 * (QSFP2_INT / 64)),
9425                           cce_int_mask);
9426         }
9427
9428         qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9429         /* Clear current status to avoid spurious interrupts */
9430         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9431                   qsfp_mask);
9432         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9433                   qsfp_mask);
9434
9435         set_qsfp_int_n(ppd, 0);
9436
9437         /* Handle active low nature of INT_N and MODPRST_N pins */
9438         if (qsfp_mod_present(ppd))
9439                 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9440         write_csr(dd,
9441                   dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9442                   qsfp_mask);
9443 }
9444
9445 /*
9446  * Do a one-time initialize of the LCB block.
9447  */
9448 static void init_lcb(struct hfi1_devdata *dd)
9449 {
9450         /* simulator does not correctly handle LCB cclk loopback, skip */
9451         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9452                 return;
9453
9454         /* the DC has been reset earlier in the driver load */
9455
9456         /* set LCB for cclk loopback on the port */
9457         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9458         write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9459         write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9460         write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9461         write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9462         write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9463         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9464 }
9465
9466 int bringup_serdes(struct hfi1_pportdata *ppd)
9467 {
9468         struct hfi1_devdata *dd = ppd->dd;
9469         u64 guid;
9470         int ret;
9471
9472         if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9473                 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9474
9475         guid = ppd->guid;
9476         if (!guid) {
9477                 if (dd->base_guid)
9478                         guid = dd->base_guid + ppd->port - 1;
9479                 ppd->guid = guid;
9480         }
9481
9482         /* Set linkinit_reason on power up per OPA spec */
9483         ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9484
9485         /* one-time init of the LCB */
9486         init_lcb(dd);
9487
9488         if (loopback) {
9489                 ret = init_loopback(dd);
9490                 if (ret < 0)
9491                         return ret;
9492         }
9493
9494         get_port_type(ppd);
9495         if (ppd->port_type == PORT_TYPE_QSFP) {
9496                 set_qsfp_int_n(ppd, 0);
9497                 wait_for_qsfp_init(ppd);
9498                 set_qsfp_int_n(ppd, 1);
9499         }
9500
9501         /*
9502          * Tune the SerDes to a ballpark setting for
9503          * optimal signal and bit error rate
9504          * Needs to be done before starting the link
9505          */
9506         tune_serdes(ppd);
9507
9508         return start_link(ppd);
9509 }
9510
9511 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9512 {
9513         struct hfi1_devdata *dd = ppd->dd;
9514
9515         /*
9516          * Shut down the link and keep it down.   First turn off that the
9517          * driver wants to allow the link to be up (driver_link_ready).
9518          * Then make sure the link is not automatically restarted
9519          * (link_enabled).  Cancel any pending restart.  And finally
9520          * go offline.
9521          */
9522         ppd->driver_link_ready = 0;
9523         ppd->link_enabled = 0;
9524
9525         ppd->offline_disabled_reason =
9526                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_SMA_DISABLED);
9527         set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
9528                              OPA_LINKDOWN_REASON_SMA_DISABLED);
9529         set_link_state(ppd, HLS_DN_OFFLINE);
9530
9531         /* disable the port */
9532         clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9533 }
9534
9535 static inline int init_cpu_counters(struct hfi1_devdata *dd)
9536 {
9537         struct hfi1_pportdata *ppd;
9538         int i;
9539
9540         ppd = (struct hfi1_pportdata *)(dd + 1);
9541         for (i = 0; i < dd->num_pports; i++, ppd++) {
9542                 ppd->ibport_data.rvp.rc_acks = NULL;
9543                 ppd->ibport_data.rvp.rc_qacks = NULL;
9544                 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9545                 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9546                 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9547                 if (!ppd->ibport_data.rvp.rc_acks ||
9548                     !ppd->ibport_data.rvp.rc_delayed_comp ||
9549                     !ppd->ibport_data.rvp.rc_qacks)
9550                         return -ENOMEM;
9551         }
9552
9553         return 0;
9554 }
9555
9556 static const char * const pt_names[] = {
9557         "expected",
9558         "eager",
9559         "invalid"
9560 };
9561
9562 static const char *pt_name(u32 type)
9563 {
9564         return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
9565 }
9566
9567 /*
9568  * index is the index into the receive array
9569  */
9570 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9571                   u32 type, unsigned long pa, u16 order)
9572 {
9573         u64 reg;
9574         void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
9575                               (dd->kregbase + RCV_ARRAY));
9576
9577         if (!(dd->flags & HFI1_PRESENT))
9578                 goto done;
9579
9580         if (type == PT_INVALID) {
9581                 pa = 0;
9582         } else if (type > PT_INVALID) {
9583                 dd_dev_err(dd,
9584                            "unexpected receive array type %u for index %u, not handled\n",
9585                            type, index);
9586                 goto done;
9587         }
9588
9589         hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
9590                   pt_name(type), index, pa, (unsigned long)order);
9591
9592 #define RT_ADDR_SHIFT 12        /* 4KB kernel address boundary */
9593         reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9594                 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9595                 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9596                                         << RCV_ARRAY_RT_ADDR_SHIFT;
9597         writeq(reg, base + (index * 8));
9598
9599         if (type == PT_EAGER)
9600                 /*
9601                  * Eager entries are written one-by-one so we have to push them
9602                  * after we write the entry.
9603                  */
9604                 flush_wc();
9605 done:
9606         return;
9607 }
9608
9609 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9610 {
9611         struct hfi1_devdata *dd = rcd->dd;
9612         u32 i;
9613
9614         /* this could be optimized */
9615         for (i = rcd->eager_base; i < rcd->eager_base +
9616                      rcd->egrbufs.alloced; i++)
9617                 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9618
9619         for (i = rcd->expected_base;
9620                         i < rcd->expected_base + rcd->expected_count; i++)
9621                 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9622 }
9623
9624 int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd,
9625                         struct hfi1_ctxt_info *kinfo)
9626 {
9627         kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) |
9628                 HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U);
9629         return 0;
9630 }
9631
9632 struct hfi1_message_header *hfi1_get_msgheader(
9633                                 struct hfi1_devdata *dd, __le32 *rhf_addr)
9634 {
9635         u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
9636
9637         return (struct hfi1_message_header *)
9638                 (rhf_addr - dd->rhf_offset + offset);
9639 }
9640
9641 static const char * const ib_cfg_name_strings[] = {
9642         "HFI1_IB_CFG_LIDLMC",
9643         "HFI1_IB_CFG_LWID_DG_ENB",
9644         "HFI1_IB_CFG_LWID_ENB",
9645         "HFI1_IB_CFG_LWID",
9646         "HFI1_IB_CFG_SPD_ENB",
9647         "HFI1_IB_CFG_SPD",
9648         "HFI1_IB_CFG_RXPOL_ENB",
9649         "HFI1_IB_CFG_LREV_ENB",
9650         "HFI1_IB_CFG_LINKLATENCY",
9651         "HFI1_IB_CFG_HRTBT",
9652         "HFI1_IB_CFG_OP_VLS",
9653         "HFI1_IB_CFG_VL_HIGH_CAP",
9654         "HFI1_IB_CFG_VL_LOW_CAP",
9655         "HFI1_IB_CFG_OVERRUN_THRESH",
9656         "HFI1_IB_CFG_PHYERR_THRESH",
9657         "HFI1_IB_CFG_LINKDEFAULT",
9658         "HFI1_IB_CFG_PKEYS",
9659         "HFI1_IB_CFG_MTU",
9660         "HFI1_IB_CFG_LSTATE",
9661         "HFI1_IB_CFG_VL_HIGH_LIMIT",
9662         "HFI1_IB_CFG_PMA_TICKS",
9663         "HFI1_IB_CFG_PORT"
9664 };
9665
9666 static const char *ib_cfg_name(int which)
9667 {
9668         if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
9669                 return "invalid";
9670         return ib_cfg_name_strings[which];
9671 }
9672
9673 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
9674 {
9675         struct hfi1_devdata *dd = ppd->dd;
9676         int val = 0;
9677
9678         switch (which) {
9679         case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
9680                 val = ppd->link_width_enabled;
9681                 break;
9682         case HFI1_IB_CFG_LWID: /* currently active Link-width */
9683                 val = ppd->link_width_active;
9684                 break;
9685         case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
9686                 val = ppd->link_speed_enabled;
9687                 break;
9688         case HFI1_IB_CFG_SPD: /* current Link speed */
9689                 val = ppd->link_speed_active;
9690                 break;
9691
9692         case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
9693         case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
9694         case HFI1_IB_CFG_LINKLATENCY:
9695                 goto unimplemented;
9696
9697         case HFI1_IB_CFG_OP_VLS:
9698                 val = ppd->vls_operational;
9699                 break;
9700         case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
9701                 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
9702                 break;
9703         case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
9704                 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
9705                 break;
9706         case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
9707                 val = ppd->overrun_threshold;
9708                 break;
9709         case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
9710                 val = ppd->phy_error_threshold;
9711                 break;
9712         case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
9713                 val = dd->link_default;
9714                 break;
9715
9716         case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
9717         case HFI1_IB_CFG_PMA_TICKS:
9718         default:
9719 unimplemented:
9720                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
9721                         dd_dev_info(
9722                                 dd,
9723                                 "%s: which %s: not implemented\n",
9724                                 __func__,
9725                                 ib_cfg_name(which));
9726                 break;
9727         }
9728
9729         return val;
9730 }
9731
9732 /*
9733  * The largest MAD packet size.
9734  */
9735 #define MAX_MAD_PACKET 2048
9736
9737 /*
9738  * Return the maximum header bytes that can go on the _wire_
9739  * for this device. This count includes the ICRC which is
9740  * not part of the packet held in memory but it is appended
9741  * by the HW.
9742  * This is dependent on the device's receive header entry size.
9743  * HFI allows this to be set per-receive context, but the
9744  * driver presently enforces a global value.
9745  */
9746 u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
9747 {
9748         /*
9749          * The maximum non-payload (MTU) bytes in LRH.PktLen are
9750          * the Receive Header Entry Size minus the PBC (or RHF) size
9751          * plus one DW for the ICRC appended by HW.
9752          *
9753          * dd->rcd[0].rcvhdrqentsize is in DW.
9754          * We use rcd[0] as all context will have the same value. Also,
9755          * the first kernel context would have been allocated by now so
9756          * we are guaranteed a valid value.
9757          */
9758         return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
9759 }
9760
9761 /*
9762  * Set Send Length
9763  * @ppd - per port data
9764  *
9765  * Set the MTU by limiting how many DWs may be sent.  The SendLenCheck*
9766  * registers compare against LRH.PktLen, so use the max bytes included
9767  * in the LRH.
9768  *
9769  * This routine changes all VL values except VL15, which it maintains at
9770  * the same value.
9771  */
9772 static void set_send_length(struct hfi1_pportdata *ppd)
9773 {
9774         struct hfi1_devdata *dd = ppd->dd;
9775         u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
9776         u32 maxvlmtu = dd->vld[15].mtu;
9777         u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
9778                               & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
9779                 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
9780         int i;
9781         u32 thres;
9782
9783         for (i = 0; i < ppd->vls_supported; i++) {
9784                 if (dd->vld[i].mtu > maxvlmtu)
9785                         maxvlmtu = dd->vld[i].mtu;
9786                 if (i <= 3)
9787                         len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
9788                                  & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
9789                                 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
9790                 else
9791                         len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
9792                                  & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
9793                                 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
9794         }
9795         write_csr(dd, SEND_LEN_CHECK0, len1);
9796         write_csr(dd, SEND_LEN_CHECK1, len2);
9797         /* adjust kernel credit return thresholds based on new MTUs */
9798         /* all kernel receive contexts have the same hdrqentsize */
9799         for (i = 0; i < ppd->vls_supported; i++) {
9800                 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
9801                             sc_mtu_to_threshold(dd->vld[i].sc,
9802                                                 dd->vld[i].mtu,
9803                                                 dd->rcd[0]->rcvhdrqentsize));
9804                 sc_set_cr_threshold(dd->vld[i].sc, thres);
9805         }
9806         thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
9807                     sc_mtu_to_threshold(dd->vld[15].sc,
9808                                         dd->vld[15].mtu,
9809                                         dd->rcd[0]->rcvhdrqentsize));
9810         sc_set_cr_threshold(dd->vld[15].sc, thres);
9811
9812         /* Adjust maximum MTU for the port in DC */
9813         dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
9814                 (ilog2(maxvlmtu >> 8) + 1);
9815         len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
9816         len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
9817         len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
9818                 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
9819         write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
9820 }
9821
9822 static void set_lidlmc(struct hfi1_pportdata *ppd)
9823 {
9824         int i;
9825         u64 sreg = 0;
9826         struct hfi1_devdata *dd = ppd->dd;
9827         u32 mask = ~((1U << ppd->lmc) - 1);
9828         u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
9829
9830         if (dd->hfi1_snoop.mode_flag)
9831                 dd_dev_info(dd, "Set lid/lmc while snooping");
9832
9833         c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
9834                 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
9835         c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
9836                         << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
9837               ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
9838                         << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
9839         write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
9840
9841         /*
9842          * Iterate over all the send contexts and set their SLID check
9843          */
9844         sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
9845                         SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
9846                (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
9847                         SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
9848
9849         for (i = 0; i < dd->chip_send_contexts; i++) {
9850                 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
9851                           i, (u32)sreg);
9852                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
9853         }
9854
9855         /* Now we have to do the same thing for the sdma engines */
9856         sdma_update_lmc(dd, mask, ppd->lid);
9857 }
9858
9859 static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
9860 {
9861         unsigned long timeout;
9862         u32 curr_state;
9863
9864         timeout = jiffies + msecs_to_jiffies(msecs);
9865         while (1) {
9866                 curr_state = read_physical_state(dd);
9867                 if (curr_state == state)
9868                         break;
9869                 if (time_after(jiffies, timeout)) {
9870                         dd_dev_err(dd,
9871                                    "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
9872                                    state, curr_state);
9873                         return -ETIMEDOUT;
9874                 }
9875                 usleep_range(1950, 2050); /* sleep 2ms-ish */
9876         }
9877
9878         return 0;
9879 }
9880
9881 /*
9882  * Helper for set_link_state().  Do not call except from that routine.
9883  * Expects ppd->hls_mutex to be held.
9884  *
9885  * @rem_reason value to be sent to the neighbor
9886  *
9887  * LinkDownReasons only set if transition succeeds.
9888  */
9889 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
9890 {
9891         struct hfi1_devdata *dd = ppd->dd;
9892         u32 pstate, previous_state;
9893         u32 last_local_state;
9894         u32 last_remote_state;
9895         int ret;
9896         int do_transition;
9897         int do_wait;
9898
9899         previous_state = ppd->host_link_state;
9900         ppd->host_link_state = HLS_GOING_OFFLINE;
9901         pstate = read_physical_state(dd);
9902         if (pstate == PLS_OFFLINE) {
9903                 do_transition = 0;      /* in right state */
9904                 do_wait = 0;            /* ...no need to wait */
9905         } else if ((pstate & 0xff) == PLS_OFFLINE) {
9906                 do_transition = 0;      /* in an offline transient state */
9907                 do_wait = 1;            /* ...wait for it to settle */
9908         } else {
9909                 do_transition = 1;      /* need to move to offline */
9910                 do_wait = 1;            /* ...will need to wait */
9911         }
9912
9913         if (do_transition) {
9914                 ret = set_physical_link_state(dd,
9915                                               (rem_reason << 8) | PLS_OFFLINE);
9916
9917                 if (ret != HCMD_SUCCESS) {
9918                         dd_dev_err(dd,
9919                                    "Failed to transition to Offline link state, return %d\n",
9920                                    ret);
9921                         return -EINVAL;
9922                 }
9923                 if (ppd->offline_disabled_reason ==
9924                                 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
9925                         ppd->offline_disabled_reason =
9926                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
9927         }
9928
9929         if (do_wait) {
9930                 /* it can take a while for the link to go down */
9931                 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
9932                 if (ret < 0)
9933                         return ret;
9934         }
9935
9936         /* make sure the logical state is also down */
9937         wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
9938
9939         /*
9940          * Now in charge of LCB - must be after the physical state is
9941          * offline.quiet and before host_link_state is changed.
9942          */
9943         set_host_lcb_access(dd);
9944         write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
9945         ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
9946
9947         if (ppd->port_type == PORT_TYPE_QSFP &&
9948             ppd->qsfp_info.limiting_active &&
9949             qsfp_mod_present(ppd)) {
9950                 int ret;
9951
9952                 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
9953                 if (ret == 0) {
9954                         set_qsfp_tx(ppd, 0);
9955                         release_chip_resource(dd, qsfp_resource(dd));
9956                 } else {
9957                         /* not fatal, but should warn */
9958                         dd_dev_err(dd,
9959                                    "Unable to acquire lock to turn off QSFP TX\n");
9960                 }
9961         }
9962
9963         /*
9964          * The LNI has a mandatory wait time after the physical state
9965          * moves to Offline.Quiet.  The wait time may be different
9966          * depending on how the link went down.  The 8051 firmware
9967          * will observe the needed wait time and only move to ready
9968          * when that is completed.  The largest of the quiet timeouts
9969          * is 6s, so wait that long and then at least 0.5s more for
9970          * other transitions, and another 0.5s for a buffer.
9971          */
9972         ret = wait_fm_ready(dd, 7000);
9973         if (ret) {
9974                 dd_dev_err(dd,
9975                            "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
9976                 /* state is really offline, so make it so */
9977                 ppd->host_link_state = HLS_DN_OFFLINE;
9978                 return ret;
9979         }
9980
9981         /*
9982          * The state is now offline and the 8051 is ready to accept host
9983          * requests.
9984          *      - change our state
9985          *      - notify others if we were previously in a linkup state
9986          */
9987         ppd->host_link_state = HLS_DN_OFFLINE;
9988         if (previous_state & HLS_UP) {
9989                 /* went down while link was up */
9990                 handle_linkup_change(dd, 0);
9991         } else if (previous_state
9992                         & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
9993                 /* went down while attempting link up */
9994                 /* byte 1 of last_*_state is the failure reason */
9995                 read_last_local_state(dd, &last_local_state);
9996                 read_last_remote_state(dd, &last_remote_state);
9997                 dd_dev_err(dd,
9998                            "LNI failure last states: local 0x%08x, remote 0x%08x\n",
9999                            last_local_state, last_remote_state);
10000         }
10001
10002         /* the active link width (downgrade) is 0 on link down */
10003         ppd->link_width_active = 0;
10004         ppd->link_width_downgrade_tx_active = 0;
10005         ppd->link_width_downgrade_rx_active = 0;
10006         ppd->current_egress_rate = 0;
10007         return 0;
10008 }
10009
10010 /* return the link state name */
10011 static const char *link_state_name(u32 state)
10012 {
10013         const char *name;
10014         int n = ilog2(state);
10015         static const char * const names[] = {
10016                 [__HLS_UP_INIT_BP]       = "INIT",
10017                 [__HLS_UP_ARMED_BP]      = "ARMED",
10018                 [__HLS_UP_ACTIVE_BP]     = "ACTIVE",
10019                 [__HLS_DN_DOWNDEF_BP]    = "DOWNDEF",
10020                 [__HLS_DN_POLL_BP]       = "POLL",
10021                 [__HLS_DN_DISABLE_BP]    = "DISABLE",
10022                 [__HLS_DN_OFFLINE_BP]    = "OFFLINE",
10023                 [__HLS_VERIFY_CAP_BP]    = "VERIFY_CAP",
10024                 [__HLS_GOING_UP_BP]      = "GOING_UP",
10025                 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10026                 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10027         };
10028
10029         name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10030         return name ? name : "unknown";
10031 }
10032
10033 /* return the link state reason name */
10034 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10035 {
10036         if (state == HLS_UP_INIT) {
10037                 switch (ppd->linkinit_reason) {
10038                 case OPA_LINKINIT_REASON_LINKUP:
10039                         return "(LINKUP)";
10040                 case OPA_LINKINIT_REASON_FLAPPING:
10041                         return "(FLAPPING)";
10042                 case OPA_LINKINIT_OUTSIDE_POLICY:
10043                         return "(OUTSIDE_POLICY)";
10044                 case OPA_LINKINIT_QUARANTINED:
10045                         return "(QUARANTINED)";
10046                 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10047                         return "(INSUFIC_CAPABILITY)";
10048                 default:
10049                         break;
10050                 }
10051         }
10052         return "";
10053 }
10054
10055 /*
10056  * driver_physical_state - convert the driver's notion of a port's
10057  * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
10058  * Return -1 (converted to a u32) to indicate error.
10059  */
10060 u32 driver_physical_state(struct hfi1_pportdata *ppd)
10061 {
10062         switch (ppd->host_link_state) {
10063         case HLS_UP_INIT:
10064         case HLS_UP_ARMED:
10065         case HLS_UP_ACTIVE:
10066                 return IB_PORTPHYSSTATE_LINKUP;
10067         case HLS_DN_POLL:
10068                 return IB_PORTPHYSSTATE_POLLING;
10069         case HLS_DN_DISABLE:
10070                 return IB_PORTPHYSSTATE_DISABLED;
10071         case HLS_DN_OFFLINE:
10072                 return OPA_PORTPHYSSTATE_OFFLINE;
10073         case HLS_VERIFY_CAP:
10074                 return IB_PORTPHYSSTATE_POLLING;
10075         case HLS_GOING_UP:
10076                 return IB_PORTPHYSSTATE_POLLING;
10077         case HLS_GOING_OFFLINE:
10078                 return OPA_PORTPHYSSTATE_OFFLINE;
10079         case HLS_LINK_COOLDOWN:
10080                 return OPA_PORTPHYSSTATE_OFFLINE;
10081         case HLS_DN_DOWNDEF:
10082         default:
10083                 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10084                            ppd->host_link_state);
10085                 return  -1;
10086         }
10087 }
10088
10089 /*
10090  * driver_logical_state - convert the driver's notion of a port's
10091  * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
10092  * (converted to a u32) to indicate error.
10093  */
10094 u32 driver_logical_state(struct hfi1_pportdata *ppd)
10095 {
10096         if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
10097                 return IB_PORT_DOWN;
10098
10099         switch (ppd->host_link_state & HLS_UP) {
10100         case HLS_UP_INIT:
10101                 return IB_PORT_INIT;
10102         case HLS_UP_ARMED:
10103                 return IB_PORT_ARMED;
10104         case HLS_UP_ACTIVE:
10105                 return IB_PORT_ACTIVE;
10106         default:
10107                 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10108                            ppd->host_link_state);
10109         return -1;
10110         }
10111 }
10112
10113 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10114                           u8 neigh_reason, u8 rem_reason)
10115 {
10116         if (ppd->local_link_down_reason.latest == 0 &&
10117             ppd->neigh_link_down_reason.latest == 0) {
10118                 ppd->local_link_down_reason.latest = lcl_reason;
10119                 ppd->neigh_link_down_reason.latest = neigh_reason;
10120                 ppd->remote_link_down_reason = rem_reason;
10121         }
10122 }
10123
10124 /*
10125  * Change the physical and/or logical link state.
10126  *
10127  * Do not call this routine while inside an interrupt.  It contains
10128  * calls to routines that can take multiple seconds to finish.
10129  *
10130  * Returns 0 on success, -errno on failure.
10131  */
10132 int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10133 {
10134         struct hfi1_devdata *dd = ppd->dd;
10135         struct ib_event event = {.device = NULL};
10136         int ret1, ret = 0;
10137         int orig_new_state, poll_bounce;
10138
10139         mutex_lock(&ppd->hls_lock);
10140
10141         orig_new_state = state;
10142         if (state == HLS_DN_DOWNDEF)
10143                 state = dd->link_default;
10144
10145         /* interpret poll -> poll as a link bounce */
10146         poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10147                       state == HLS_DN_POLL;
10148
10149         dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
10150                     link_state_name(ppd->host_link_state),
10151                     link_state_name(orig_new_state),
10152                     poll_bounce ? "(bounce) " : "",
10153                     link_state_reason_name(ppd, state));
10154
10155         /*
10156          * If we're going to a (HLS_*) link state that implies the logical
10157          * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10158          * reset is_sm_config_started to 0.
10159          */
10160         if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10161                 ppd->is_sm_config_started = 0;
10162
10163         /*
10164          * Do nothing if the states match.  Let a poll to poll link bounce
10165          * go through.
10166          */
10167         if (ppd->host_link_state == state && !poll_bounce)
10168                 goto done;
10169
10170         switch (state) {
10171         case HLS_UP_INIT:
10172                 if (ppd->host_link_state == HLS_DN_POLL &&
10173                     (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
10174                         /*
10175                          * Quick link up jumps from polling to here.
10176                          *
10177                          * Whether in normal or loopback mode, the
10178                          * simulator jumps from polling to link up.
10179                          * Accept that here.
10180                          */
10181                         /* OK */
10182                 } else if (ppd->host_link_state != HLS_GOING_UP) {
10183                         goto unexpected;
10184                 }
10185
10186                 ppd->host_link_state = HLS_UP_INIT;
10187                 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10188                 if (ret) {
10189                         /* logical state didn't change, stay at going_up */
10190                         ppd->host_link_state = HLS_GOING_UP;
10191                         dd_dev_err(dd,
10192                                    "%s: logical state did not change to INIT\n",
10193                                    __func__);
10194                 } else {
10195                         /* clear old transient LINKINIT_REASON code */
10196                         if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10197                                 ppd->linkinit_reason =
10198                                         OPA_LINKINIT_REASON_LINKUP;
10199
10200                         /* enable the port */
10201                         add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10202
10203                         handle_linkup_change(dd, 1);
10204                 }
10205                 break;
10206         case HLS_UP_ARMED:
10207                 if (ppd->host_link_state != HLS_UP_INIT)
10208                         goto unexpected;
10209
10210                 ppd->host_link_state = HLS_UP_ARMED;
10211                 set_logical_state(dd, LSTATE_ARMED);
10212                 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10213                 if (ret) {
10214                         /* logical state didn't change, stay at init */
10215                         ppd->host_link_state = HLS_UP_INIT;
10216                         dd_dev_err(dd,
10217                                    "%s: logical state did not change to ARMED\n",
10218                                    __func__);
10219                 }
10220                 /*
10221                  * The simulator does not currently implement SMA messages,
10222                  * so neighbor_normal is not set.  Set it here when we first
10223                  * move to Armed.
10224                  */
10225                 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10226                         ppd->neighbor_normal = 1;
10227                 break;
10228         case HLS_UP_ACTIVE:
10229                 if (ppd->host_link_state != HLS_UP_ARMED)
10230                         goto unexpected;
10231
10232                 ppd->host_link_state = HLS_UP_ACTIVE;
10233                 set_logical_state(dd, LSTATE_ACTIVE);
10234                 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10235                 if (ret) {
10236                         /* logical state didn't change, stay at armed */
10237                         ppd->host_link_state = HLS_UP_ARMED;
10238                         dd_dev_err(dd,
10239                                    "%s: logical state did not change to ACTIVE\n",
10240                                    __func__);
10241                 } else {
10242                         /* tell all engines to go running */
10243                         sdma_all_running(dd);
10244
10245                         /* Signal the IB layer that the port has went active */
10246                         event.device = &dd->verbs_dev.rdi.ibdev;
10247                         event.element.port_num = ppd->port;
10248                         event.event = IB_EVENT_PORT_ACTIVE;
10249                 }
10250                 break;
10251         case HLS_DN_POLL:
10252                 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10253                      ppd->host_link_state == HLS_DN_OFFLINE) &&
10254                     dd->dc_shutdown)
10255                         dc_start(dd);
10256                 /* Hand LED control to the DC */
10257                 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10258
10259                 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10260                         u8 tmp = ppd->link_enabled;
10261
10262                         ret = goto_offline(ppd, ppd->remote_link_down_reason);
10263                         if (ret) {
10264                                 ppd->link_enabled = tmp;
10265                                 break;
10266                         }
10267                         ppd->remote_link_down_reason = 0;
10268
10269                         if (ppd->driver_link_ready)
10270                                 ppd->link_enabled = 1;
10271                 }
10272
10273                 set_all_slowpath(ppd->dd);
10274                 ret = set_local_link_attributes(ppd);
10275                 if (ret)
10276                         break;
10277
10278                 ppd->port_error_action = 0;
10279                 ppd->host_link_state = HLS_DN_POLL;
10280
10281                 if (quick_linkup) {
10282                         /* quick linkup does not go into polling */
10283                         ret = do_quick_linkup(dd);
10284                 } else {
10285                         ret1 = set_physical_link_state(dd, PLS_POLLING);
10286                         if (ret1 != HCMD_SUCCESS) {
10287                                 dd_dev_err(dd,
10288                                            "Failed to transition to Polling link state, return 0x%x\n",
10289                                            ret1);
10290                                 ret = -EINVAL;
10291                         }
10292                 }
10293                 ppd->offline_disabled_reason =
10294                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
10295                 /*
10296                  * If an error occurred above, go back to offline.  The
10297                  * caller may reschedule another attempt.
10298                  */
10299                 if (ret)
10300                         goto_offline(ppd, 0);
10301                 break;
10302         case HLS_DN_DISABLE:
10303                 /* link is disabled */
10304                 ppd->link_enabled = 0;
10305
10306                 /* allow any state to transition to disabled */
10307
10308                 /* must transition to offline first */
10309                 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10310                         ret = goto_offline(ppd, ppd->remote_link_down_reason);
10311                         if (ret)
10312                                 break;
10313                         ppd->remote_link_down_reason = 0;
10314                 }
10315
10316                 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10317                 if (ret1 != HCMD_SUCCESS) {
10318                         dd_dev_err(dd,
10319                                    "Failed to transition to Disabled link state, return 0x%x\n",
10320                                    ret1);
10321                         ret = -EINVAL;
10322                         break;
10323                 }
10324                 ppd->host_link_state = HLS_DN_DISABLE;
10325                 dc_shutdown(dd);
10326                 break;
10327         case HLS_DN_OFFLINE:
10328                 if (ppd->host_link_state == HLS_DN_DISABLE)
10329                         dc_start(dd);
10330
10331                 /* allow any state to transition to offline */
10332                 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10333                 if (!ret)
10334                         ppd->remote_link_down_reason = 0;
10335                 break;
10336         case HLS_VERIFY_CAP:
10337                 if (ppd->host_link_state != HLS_DN_POLL)
10338                         goto unexpected;
10339                 ppd->host_link_state = HLS_VERIFY_CAP;
10340                 break;
10341         case HLS_GOING_UP:
10342                 if (ppd->host_link_state != HLS_VERIFY_CAP)
10343                         goto unexpected;
10344
10345                 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10346                 if (ret1 != HCMD_SUCCESS) {
10347                         dd_dev_err(dd,
10348                                    "Failed to transition to link up state, return 0x%x\n",
10349                                    ret1);
10350                         ret = -EINVAL;
10351                         break;
10352                 }
10353                 ppd->host_link_state = HLS_GOING_UP;
10354                 break;
10355
10356         case HLS_GOING_OFFLINE:         /* transient within goto_offline() */
10357         case HLS_LINK_COOLDOWN:         /* transient within goto_offline() */
10358         default:
10359                 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
10360                             __func__, state);
10361                 ret = -EINVAL;
10362                 break;
10363         }
10364
10365         goto done;
10366
10367 unexpected:
10368         dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
10369                    __func__, link_state_name(ppd->host_link_state),
10370                    link_state_name(state));
10371         ret = -EINVAL;
10372
10373 done:
10374         mutex_unlock(&ppd->hls_lock);
10375
10376         if (event.device)
10377                 ib_dispatch_event(&event);
10378
10379         return ret;
10380 }
10381
10382 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10383 {
10384         u64 reg;
10385         int ret = 0;
10386
10387         switch (which) {
10388         case HFI1_IB_CFG_LIDLMC:
10389                 set_lidlmc(ppd);
10390                 break;
10391         case HFI1_IB_CFG_VL_HIGH_LIMIT:
10392                 /*
10393                  * The VL Arbitrator high limit is sent in units of 4k
10394                  * bytes, while HFI stores it in units of 64 bytes.
10395                  */
10396                 val *= 4096 / 64;
10397                 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10398                         << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10399                 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10400                 break;
10401         case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10402                 /* HFI only supports POLL as the default link down state */
10403                 if (val != HLS_DN_POLL)
10404                         ret = -EINVAL;
10405                 break;
10406         case HFI1_IB_CFG_OP_VLS:
10407                 if (ppd->vls_operational != val) {
10408                         ppd->vls_operational = val;
10409                         if (!ppd->port)
10410                                 ret = -EINVAL;
10411                 }
10412                 break;
10413         /*
10414          * For link width, link width downgrade, and speed enable, always AND
10415          * the setting with what is actually supported.  This has two benefits.
10416          * First, enabled can't have unsupported values, no matter what the
10417          * SM or FM might want.  Second, the ALL_SUPPORTED wildcards that mean
10418          * "fill in with your supported value" have all the bits in the
10419          * field set, so simply ANDing with supported has the desired result.
10420          */
10421         case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
10422                 ppd->link_width_enabled = val & ppd->link_width_supported;
10423                 break;
10424         case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
10425                 ppd->link_width_downgrade_enabled =
10426                                 val & ppd->link_width_downgrade_supported;
10427                 break;
10428         case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10429                 ppd->link_speed_enabled = val & ppd->link_speed_supported;
10430                 break;
10431         case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10432                 /*
10433                  * HFI does not follow IB specs, save this value
10434                  * so we can report it, if asked.
10435                  */
10436                 ppd->overrun_threshold = val;
10437                 break;
10438         case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10439                 /*
10440                  * HFI does not follow IB specs, save this value
10441                  * so we can report it, if asked.
10442                  */
10443                 ppd->phy_error_threshold = val;
10444                 break;
10445
10446         case HFI1_IB_CFG_MTU:
10447                 set_send_length(ppd);
10448                 break;
10449
10450         case HFI1_IB_CFG_PKEYS:
10451                 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10452                         set_partition_keys(ppd);
10453                 break;
10454
10455         default:
10456                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10457                         dd_dev_info(ppd->dd,
10458                                     "%s: which %s, val 0x%x: not implemented\n",
10459                                     __func__, ib_cfg_name(which), val);
10460                 break;
10461         }
10462         return ret;
10463 }
10464
10465 /* begin functions related to vl arbitration table caching */
10466 static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
10467 {
10468         int i;
10469
10470         BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10471                         VL_ARB_LOW_PRIO_TABLE_SIZE);
10472         BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10473                         VL_ARB_HIGH_PRIO_TABLE_SIZE);
10474
10475         /*
10476          * Note that we always return values directly from the
10477          * 'vl_arb_cache' (and do no CSR reads) in response to a
10478          * 'Get(VLArbTable)'. This is obviously correct after a
10479          * 'Set(VLArbTable)', since the cache will then be up to
10480          * date. But it's also correct prior to any 'Set(VLArbTable)'
10481          * since then both the cache, and the relevant h/w registers
10482          * will be zeroed.
10483          */
10484
10485         for (i = 0; i < MAX_PRIO_TABLE; i++)
10486                 spin_lock_init(&ppd->vl_arb_cache[i].lock);
10487 }
10488
10489 /*
10490  * vl_arb_lock_cache
10491  *
10492  * All other vl_arb_* functions should be called only after locking
10493  * the cache.
10494  */
10495 static inline struct vl_arb_cache *
10496 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
10497 {
10498         if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
10499                 return NULL;
10500         spin_lock(&ppd->vl_arb_cache[idx].lock);
10501         return &ppd->vl_arb_cache[idx];
10502 }
10503
10504 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
10505 {
10506         spin_unlock(&ppd->vl_arb_cache[idx].lock);
10507 }
10508
10509 static void vl_arb_get_cache(struct vl_arb_cache *cache,
10510                              struct ib_vl_weight_elem *vl)
10511 {
10512         memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
10513 }
10514
10515 static void vl_arb_set_cache(struct vl_arb_cache *cache,
10516                              struct ib_vl_weight_elem *vl)
10517 {
10518         memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10519 }
10520
10521 static int vl_arb_match_cache(struct vl_arb_cache *cache,
10522                               struct ib_vl_weight_elem *vl)
10523 {
10524         return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
10525 }
10526
10527 /* end functions related to vl arbitration table caching */
10528
10529 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
10530                           u32 size, struct ib_vl_weight_elem *vl)
10531 {
10532         struct hfi1_devdata *dd = ppd->dd;
10533         u64 reg;
10534         unsigned int i, is_up = 0;
10535         int drain, ret = 0;
10536
10537         mutex_lock(&ppd->hls_lock);
10538
10539         if (ppd->host_link_state & HLS_UP)
10540                 is_up = 1;
10541
10542         drain = !is_ax(dd) && is_up;
10543
10544         if (drain)
10545                 /*
10546                  * Before adjusting VL arbitration weights, empty per-VL
10547                  * FIFOs, otherwise a packet whose VL weight is being
10548                  * set to 0 could get stuck in a FIFO with no chance to
10549                  * egress.
10550                  */
10551                 ret = stop_drain_data_vls(dd);
10552
10553         if (ret) {
10554                 dd_dev_err(
10555                         dd,
10556                         "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
10557                         __func__);
10558                 goto err;
10559         }
10560
10561         for (i = 0; i < size; i++, vl++) {
10562                 /*
10563                  * NOTE: The low priority shift and mask are used here, but
10564                  * they are the same for both the low and high registers.
10565                  */
10566                 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
10567                                 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
10568                       | (((u64)vl->weight
10569                                 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
10570                                 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
10571                 write_csr(dd, target + (i * 8), reg);
10572         }
10573         pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
10574
10575         if (drain)
10576                 open_fill_data_vls(dd); /* reopen all VLs */
10577
10578 err:
10579         mutex_unlock(&ppd->hls_lock);
10580
10581         return ret;
10582 }
10583
10584 /*
10585  * Read one credit merge VL register.
10586  */
10587 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
10588                            struct vl_limit *vll)
10589 {
10590         u64 reg = read_csr(dd, csr);
10591
10592         vll->dedicated = cpu_to_be16(
10593                 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
10594                 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
10595         vll->shared = cpu_to_be16(
10596                 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
10597                 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
10598 }
10599
10600 /*
10601  * Read the current credit merge limits.
10602  */
10603 static int get_buffer_control(struct hfi1_devdata *dd,
10604                               struct buffer_control *bc, u16 *overall_limit)
10605 {
10606         u64 reg;
10607         int i;
10608
10609         /* not all entries are filled in */
10610         memset(bc, 0, sizeof(*bc));
10611
10612         /* OPA and HFI have a 1-1 mapping */
10613         for (i = 0; i < TXE_NUM_DATA_VL; i++)
10614                 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
10615
10616         /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
10617         read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
10618
10619         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10620         bc->overall_shared_limit = cpu_to_be16(
10621                 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
10622                 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
10623         if (overall_limit)
10624                 *overall_limit = (reg
10625                         >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
10626                         & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
10627         return sizeof(struct buffer_control);
10628 }
10629
10630 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10631 {
10632         u64 reg;
10633         int i;
10634
10635         /* each register contains 16 SC->VLnt mappings, 4 bits each */
10636         reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
10637         for (i = 0; i < sizeof(u64); i++) {
10638                 u8 byte = *(((u8 *)&reg) + i);
10639
10640                 dp->vlnt[2 * i] = byte & 0xf;
10641                 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
10642         }
10643
10644         reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
10645         for (i = 0; i < sizeof(u64); i++) {
10646                 u8 byte = *(((u8 *)&reg) + i);
10647
10648                 dp->vlnt[16 + (2 * i)] = byte & 0xf;
10649                 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
10650         }
10651         return sizeof(struct sc2vlnt);
10652 }
10653
10654 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
10655                               struct ib_vl_weight_elem *vl)
10656 {
10657         unsigned int i;
10658
10659         for (i = 0; i < nelems; i++, vl++) {
10660                 vl->vl = 0xf;
10661                 vl->weight = 0;
10662         }
10663 }
10664
10665 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
10666 {
10667         write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
10668                   DC_SC_VL_VAL(15_0,
10669                                0, dp->vlnt[0] & 0xf,
10670                                1, dp->vlnt[1] & 0xf,
10671                                2, dp->vlnt[2] & 0xf,
10672                                3, dp->vlnt[3] & 0xf,
10673                                4, dp->vlnt[4] & 0xf,
10674                                5, dp->vlnt[5] & 0xf,
10675                                6, dp->vlnt[6] & 0xf,
10676                                7, dp->vlnt[7] & 0xf,
10677                                8, dp->vlnt[8] & 0xf,
10678                                9, dp->vlnt[9] & 0xf,
10679                                10, dp->vlnt[10] & 0xf,
10680                                11, dp->vlnt[11] & 0xf,
10681                                12, dp->vlnt[12] & 0xf,
10682                                13, dp->vlnt[13] & 0xf,
10683                                14, dp->vlnt[14] & 0xf,
10684                                15, dp->vlnt[15] & 0xf));
10685         write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
10686                   DC_SC_VL_VAL(31_16,
10687                                16, dp->vlnt[16] & 0xf,
10688                                17, dp->vlnt[17] & 0xf,
10689                                18, dp->vlnt[18] & 0xf,
10690                                19, dp->vlnt[19] & 0xf,
10691                                20, dp->vlnt[20] & 0xf,
10692                                21, dp->vlnt[21] & 0xf,
10693                                22, dp->vlnt[22] & 0xf,
10694                                23, dp->vlnt[23] & 0xf,
10695                                24, dp->vlnt[24] & 0xf,
10696                                25, dp->vlnt[25] & 0xf,
10697                                26, dp->vlnt[26] & 0xf,
10698                                27, dp->vlnt[27] & 0xf,
10699                                28, dp->vlnt[28] & 0xf,
10700                                29, dp->vlnt[29] & 0xf,
10701                                30, dp->vlnt[30] & 0xf,
10702                                31, dp->vlnt[31] & 0xf));
10703 }
10704
10705 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
10706                         u16 limit)
10707 {
10708         if (limit != 0)
10709                 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
10710                             what, (int)limit, idx);
10711 }
10712
10713 /* change only the shared limit portion of SendCmGLobalCredit */
10714 static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
10715 {
10716         u64 reg;
10717
10718         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10719         reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
10720         reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
10721         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10722 }
10723
10724 /* change only the total credit limit portion of SendCmGLobalCredit */
10725 static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
10726 {
10727         u64 reg;
10728
10729         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
10730         reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
10731         reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
10732         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
10733 }
10734
10735 /* set the given per-VL shared limit */
10736 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
10737 {
10738         u64 reg;
10739         u32 addr;
10740
10741         if (vl < TXE_NUM_DATA_VL)
10742                 addr = SEND_CM_CREDIT_VL + (8 * vl);
10743         else
10744                 addr = SEND_CM_CREDIT_VL15;
10745
10746         reg = read_csr(dd, addr);
10747         reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
10748         reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
10749         write_csr(dd, addr, reg);
10750 }
10751
10752 /* set the given per-VL dedicated limit */
10753 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
10754 {
10755         u64 reg;
10756         u32 addr;
10757
10758         if (vl < TXE_NUM_DATA_VL)
10759                 addr = SEND_CM_CREDIT_VL + (8 * vl);
10760         else
10761                 addr = SEND_CM_CREDIT_VL15;
10762
10763         reg = read_csr(dd, addr);
10764         reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
10765         reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
10766         write_csr(dd, addr, reg);
10767 }
10768
10769 /* spin until the given per-VL status mask bits clear */
10770 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
10771                                      const char *which)
10772 {
10773         unsigned long timeout;
10774         u64 reg;
10775
10776         timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
10777         while (1) {
10778                 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
10779
10780                 if (reg == 0)
10781                         return; /* success */
10782                 if (time_after(jiffies, timeout))
10783                         break;          /* timed out */
10784                 udelay(1);
10785         }
10786
10787         dd_dev_err(dd,
10788                    "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
10789                    which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
10790         /*
10791          * If this occurs, it is likely there was a credit loss on the link.
10792          * The only recovery from that is a link bounce.
10793          */
10794         dd_dev_err(dd,
10795                    "Continuing anyway.  A credit loss may occur.  Suggest a link bounce\n");
10796 }
10797
10798 /*
10799  * The number of credits on the VLs may be changed while everything
10800  * is "live", but the following algorithm must be followed due to
10801  * how the hardware is actually implemented.  In particular,
10802  * Return_Credit_Status[] is the only correct status check.
10803  *
10804  * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
10805  *     set Global_Shared_Credit_Limit = 0
10806  *     use_all_vl = 1
10807  * mask0 = all VLs that are changing either dedicated or shared limits
10808  * set Shared_Limit[mask0] = 0
10809  * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
10810  * if (changing any dedicated limit)
10811  *     mask1 = all VLs that are lowering dedicated limits
10812  *     lower Dedicated_Limit[mask1]
10813  *     spin until Return_Credit_Status[mask1] == 0
10814  *     raise Dedicated_Limits
10815  * raise Shared_Limits
10816  * raise Global_Shared_Credit_Limit
10817  *
10818  * lower = if the new limit is lower, set the limit to the new value
10819  * raise = if the new limit is higher than the current value (may be changed
10820  *      earlier in the algorithm), set the new limit to the new value
10821  */
10822 int set_buffer_control(struct hfi1_pportdata *ppd,
10823                        struct buffer_control *new_bc)
10824 {
10825         struct hfi1_devdata *dd = ppd->dd;
10826         u64 changing_mask, ld_mask, stat_mask;
10827         int change_count;
10828         int i, use_all_mask;
10829         int this_shared_changing;
10830         int vl_count = 0, ret;
10831         /*
10832          * A0: add the variable any_shared_limit_changing below and in the
10833          * algorithm above.  If removing A0 support, it can be removed.
10834          */
10835         int any_shared_limit_changing;
10836         struct buffer_control cur_bc;
10837         u8 changing[OPA_MAX_VLS];
10838         u8 lowering_dedicated[OPA_MAX_VLS];
10839         u16 cur_total;
10840         u32 new_total = 0;
10841         const u64 all_mask =
10842         SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
10843          | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
10844          | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
10845          | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
10846          | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
10847          | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
10848          | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
10849          | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
10850          | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
10851
10852 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
10853 #define NUM_USABLE_VLS 16       /* look at VL15 and less */
10854
10855         /* find the new total credits, do sanity check on unused VLs */
10856         for (i = 0; i < OPA_MAX_VLS; i++) {
10857                 if (valid_vl(i)) {
10858                         new_total += be16_to_cpu(new_bc->vl[i].dedicated);
10859                         continue;
10860                 }
10861                 nonzero_msg(dd, i, "dedicated",
10862                             be16_to_cpu(new_bc->vl[i].dedicated));
10863                 nonzero_msg(dd, i, "shared",
10864                             be16_to_cpu(new_bc->vl[i].shared));
10865                 new_bc->vl[i].dedicated = 0;
10866                 new_bc->vl[i].shared = 0;
10867         }
10868         new_total += be16_to_cpu(new_bc->overall_shared_limit);
10869
10870         /* fetch the current values */
10871         get_buffer_control(dd, &cur_bc, &cur_total);
10872
10873         /*
10874          * Create the masks we will use.
10875          */
10876         memset(changing, 0, sizeof(changing));
10877         memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
10878         /*
10879          * NOTE: Assumes that the individual VL bits are adjacent and in
10880          * increasing order
10881          */
10882         stat_mask =
10883                 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
10884         changing_mask = 0;
10885         ld_mask = 0;
10886         change_count = 0;
10887         any_shared_limit_changing = 0;
10888         for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
10889                 if (!valid_vl(i))
10890                         continue;
10891                 this_shared_changing = new_bc->vl[i].shared
10892                                                 != cur_bc.vl[i].shared;
10893                 if (this_shared_changing)
10894                         any_shared_limit_changing = 1;
10895                 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
10896                     this_shared_changing) {
10897                         changing[i] = 1;
10898                         changing_mask |= stat_mask;
10899                         change_count++;
10900                 }
10901                 if (be16_to_cpu(new_bc->vl[i].dedicated) <
10902                                         be16_to_cpu(cur_bc.vl[i].dedicated)) {
10903                         lowering_dedicated[i] = 1;
10904                         ld_mask |= stat_mask;
10905                 }
10906         }
10907
10908         /* bracket the credit change with a total adjustment */
10909         if (new_total > cur_total)
10910                 set_global_limit(dd, new_total);
10911
10912         /*
10913          * Start the credit change algorithm.
10914          */
10915         use_all_mask = 0;
10916         if ((be16_to_cpu(new_bc->overall_shared_limit) <
10917              be16_to_cpu(cur_bc.overall_shared_limit)) ||
10918             (is_ax(dd) && any_shared_limit_changing)) {
10919                 set_global_shared(dd, 0);
10920                 cur_bc.overall_shared_limit = 0;
10921                 use_all_mask = 1;
10922         }
10923
10924         for (i = 0; i < NUM_USABLE_VLS; i++) {
10925                 if (!valid_vl(i))
10926                         continue;
10927
10928                 if (changing[i]) {
10929                         set_vl_shared(dd, i, 0);
10930                         cur_bc.vl[i].shared = 0;
10931                 }
10932         }
10933
10934         wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
10935                                  "shared");
10936
10937         if (change_count > 0) {
10938                 for (i = 0; i < NUM_USABLE_VLS; i++) {
10939                         if (!valid_vl(i))
10940                                 continue;
10941
10942                         if (lowering_dedicated[i]) {
10943                                 set_vl_dedicated(dd, i,
10944                                                  be16_to_cpu(new_bc->
10945                                                              vl[i].dedicated));
10946                                 cur_bc.vl[i].dedicated =
10947                                                 new_bc->vl[i].dedicated;
10948                         }
10949                 }
10950
10951                 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
10952
10953                 /* now raise all dedicated that are going up */
10954                 for (i = 0; i < NUM_USABLE_VLS; i++) {
10955                         if (!valid_vl(i))
10956                                 continue;
10957
10958                         if (be16_to_cpu(new_bc->vl[i].dedicated) >
10959                                         be16_to_cpu(cur_bc.vl[i].dedicated))
10960                                 set_vl_dedicated(dd, i,
10961                                                  be16_to_cpu(new_bc->
10962                                                              vl[i].dedicated));
10963                 }
10964         }
10965
10966         /* next raise all shared that are going up */
10967         for (i = 0; i < NUM_USABLE_VLS; i++) {
10968                 if (!valid_vl(i))
10969                         continue;
10970
10971                 if (be16_to_cpu(new_bc->vl[i].shared) >
10972                                 be16_to_cpu(cur_bc.vl[i].shared))
10973                         set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
10974         }
10975
10976         /* finally raise the global shared */
10977         if (be16_to_cpu(new_bc->overall_shared_limit) >
10978             be16_to_cpu(cur_bc.overall_shared_limit))
10979                 set_global_shared(dd,
10980                                   be16_to_cpu(new_bc->overall_shared_limit));
10981
10982         /* bracket the credit change with a total adjustment */
10983         if (new_total < cur_total)
10984                 set_global_limit(dd, new_total);
10985
10986         /*
10987          * Determine the actual number of operational VLS using the number of
10988          * dedicated and shared credits for each VL.
10989          */
10990         if (change_count > 0) {
10991                 for (i = 0; i < TXE_NUM_DATA_VL; i++)
10992                         if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
10993                             be16_to_cpu(new_bc->vl[i].shared) > 0)
10994                                 vl_count++;
10995                 ppd->actual_vls_operational = vl_count;
10996                 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
10997                                     ppd->actual_vls_operational :
10998                                     ppd->vls_operational,
10999                                     NULL);
11000                 if (ret == 0)
11001                         ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11002                                            ppd->actual_vls_operational :
11003                                            ppd->vls_operational, NULL);
11004                 if (ret)
11005                         return ret;
11006         }
11007         return 0;
11008 }
11009
11010 /*
11011  * Read the given fabric manager table. Return the size of the
11012  * table (in bytes) on success, and a negative error code on
11013  * failure.
11014  */
11015 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11016
11017 {
11018         int size;
11019         struct vl_arb_cache *vlc;
11020
11021         switch (which) {
11022         case FM_TBL_VL_HIGH_ARB:
11023                 size = 256;
11024                 /*
11025                  * OPA specifies 128 elements (of 2 bytes each), though
11026                  * HFI supports only 16 elements in h/w.
11027                  */
11028                 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11029                 vl_arb_get_cache(vlc, t);
11030                 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11031                 break;
11032         case FM_TBL_VL_LOW_ARB:
11033                 size = 256;
11034                 /*
11035                  * OPA specifies 128 elements (of 2 bytes each), though
11036                  * HFI supports only 16 elements in h/w.
11037                  */
11038                 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11039                 vl_arb_get_cache(vlc, t);
11040                 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11041                 break;
11042         case FM_TBL_BUFFER_CONTROL:
11043                 size = get_buffer_control(ppd->dd, t, NULL);
11044                 break;
11045         case FM_TBL_SC2VLNT:
11046                 size = get_sc2vlnt(ppd->dd, t);
11047                 break;
11048         case FM_TBL_VL_PREEMPT_ELEMS:
11049                 size = 256;
11050                 /* OPA specifies 128 elements, of 2 bytes each */
11051                 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11052                 break;
11053         case FM_TBL_VL_PREEMPT_MATRIX:
11054                 size = 256;
11055                 /*
11056                  * OPA specifies that this is the same size as the VL
11057                  * arbitration tables (i.e., 256 bytes).
11058                  */
11059                 break;
11060         default:
11061                 return -EINVAL;
11062         }
11063         return size;
11064 }
11065
11066 /*
11067  * Write the given fabric manager table.
11068  */
11069 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11070 {
11071         int ret = 0;
11072         struct vl_arb_cache *vlc;
11073
11074         switch (which) {
11075         case FM_TBL_VL_HIGH_ARB:
11076                 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11077                 if (vl_arb_match_cache(vlc, t)) {
11078                         vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11079                         break;
11080                 }
11081                 vl_arb_set_cache(vlc, t);
11082                 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11083                 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11084                                      VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11085                 break;
11086         case FM_TBL_VL_LOW_ARB:
11087                 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11088                 if (vl_arb_match_cache(vlc, t)) {
11089                         vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11090                         break;
11091                 }
11092                 vl_arb_set_cache(vlc, t);
11093                 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11094                 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11095                                      VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11096                 break;
11097         case FM_TBL_BUFFER_CONTROL:
11098                 ret = set_buffer_control(ppd, t);
11099                 break;
11100         case FM_TBL_SC2VLNT:
11101                 set_sc2vlnt(ppd->dd, t);
11102                 break;
11103         default:
11104                 ret = -EINVAL;
11105         }
11106         return ret;
11107 }
11108
11109 /*
11110  * Disable all data VLs.
11111  *
11112  * Return 0 if disabled, non-zero if the VLs cannot be disabled.
11113  */
11114 static int disable_data_vls(struct hfi1_devdata *dd)
11115 {
11116         if (is_ax(dd))
11117                 return 1;
11118
11119         pio_send_control(dd, PSC_DATA_VL_DISABLE);
11120
11121         return 0;
11122 }
11123
11124 /*
11125  * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11126  * Just re-enables all data VLs (the "fill" part happens
11127  * automatically - the name was chosen for symmetry with
11128  * stop_drain_data_vls()).
11129  *
11130  * Return 0 if successful, non-zero if the VLs cannot be enabled.
11131  */
11132 int open_fill_data_vls(struct hfi1_devdata *dd)
11133 {
11134         if (is_ax(dd))
11135                 return 1;
11136
11137         pio_send_control(dd, PSC_DATA_VL_ENABLE);
11138
11139         return 0;
11140 }
11141
11142 /*
11143  * drain_data_vls() - assumes that disable_data_vls() has been called,
11144  * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11145  * engines to drop to 0.
11146  */
11147 static void drain_data_vls(struct hfi1_devdata *dd)
11148 {
11149         sc_wait(dd);
11150         sdma_wait(dd);
11151         pause_for_credit_return(dd);
11152 }
11153
11154 /*
11155  * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11156  *
11157  * Use open_fill_data_vls() to resume using data VLs.  This pair is
11158  * meant to be used like this:
11159  *
11160  * stop_drain_data_vls(dd);
11161  * // do things with per-VL resources
11162  * open_fill_data_vls(dd);
11163  */
11164 int stop_drain_data_vls(struct hfi1_devdata *dd)
11165 {
11166         int ret;
11167
11168         ret = disable_data_vls(dd);
11169         if (ret == 0)
11170                 drain_data_vls(dd);
11171
11172         return ret;
11173 }
11174
11175 /*
11176  * Convert a nanosecond time to a cclock count.  No matter how slow
11177  * the cclock, a non-zero ns will always have a non-zero result.
11178  */
11179 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11180 {
11181         u32 cclocks;
11182
11183         if (dd->icode == ICODE_FPGA_EMULATION)
11184                 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11185         else  /* simulation pretends to be ASIC */
11186                 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11187         if (ns && !cclocks)     /* if ns nonzero, must be at least 1 */
11188                 cclocks = 1;
11189         return cclocks;
11190 }
11191
11192 /*
11193  * Convert a cclock count to nanoseconds. Not matter how slow
11194  * the cclock, a non-zero cclocks will always have a non-zero result.
11195  */
11196 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11197 {
11198         u32 ns;
11199
11200         if (dd->icode == ICODE_FPGA_EMULATION)
11201                 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11202         else  /* simulation pretends to be ASIC */
11203                 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11204         if (cclocks && !ns)
11205                 ns = 1;
11206         return ns;
11207 }
11208
11209 /*
11210  * Dynamically adjust the receive interrupt timeout for a context based on
11211  * incoming packet rate.
11212  *
11213  * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11214  */
11215 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11216 {
11217         struct hfi1_devdata *dd = rcd->dd;
11218         u32 timeout = rcd->rcvavail_timeout;
11219
11220         /*
11221          * This algorithm doubles or halves the timeout depending on whether
11222          * the number of packets received in this interrupt were less than or
11223          * greater equal the interrupt count.
11224          *
11225          * The calculations below do not allow a steady state to be achieved.
11226          * Only at the endpoints it is possible to have an unchanging
11227          * timeout.
11228          */
11229         if (npkts < rcv_intr_count) {
11230                 /*
11231                  * Not enough packets arrived before the timeout, adjust
11232                  * timeout downward.
11233                  */
11234                 if (timeout < 2) /* already at minimum? */
11235                         return;
11236                 timeout >>= 1;
11237         } else {
11238                 /*
11239                  * More than enough packets arrived before the timeout, adjust
11240                  * timeout upward.
11241                  */
11242                 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11243                         return;
11244                 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11245         }
11246
11247         rcd->rcvavail_timeout = timeout;
11248         /*
11249          * timeout cannot be larger than rcv_intr_timeout_csr which has already
11250          * been verified to be in range
11251          */
11252         write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
11253                         (u64)timeout <<
11254                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11255 }
11256
11257 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11258                     u32 intr_adjust, u32 npkts)
11259 {
11260         struct hfi1_devdata *dd = rcd->dd;
11261         u64 reg;
11262         u32 ctxt = rcd->ctxt;
11263
11264         /*
11265          * Need to write timeout register before updating RcvHdrHead to ensure
11266          * that a new value is used when the HW decides to restart counting.
11267          */
11268         if (intr_adjust)
11269                 adjust_rcv_timeout(rcd, npkts);
11270         if (updegr) {
11271                 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11272                         << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11273                 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11274         }
11275         mmiowb();
11276         reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11277                 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11278                         << RCV_HDR_HEAD_HEAD_SHIFT);
11279         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11280         mmiowb();
11281 }
11282
11283 u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11284 {
11285         u32 head, tail;
11286
11287         head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11288                 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11289
11290         if (rcd->rcvhdrtail_kvaddr)
11291                 tail = get_rcvhdrtail(rcd);
11292         else
11293                 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11294
11295         return head == tail;
11296 }
11297
11298 /*
11299  * Context Control and Receive Array encoding for buffer size:
11300  *      0x0 invalid
11301  *      0x1   4 KB
11302  *      0x2   8 KB
11303  *      0x3  16 KB
11304  *      0x4  32 KB
11305  *      0x5  64 KB
11306  *      0x6 128 KB
11307  *      0x7 256 KB
11308  *      0x8 512 KB (Receive Array only)
11309  *      0x9   1 MB (Receive Array only)
11310  *      0xa   2 MB (Receive Array only)
11311  *
11312  *      0xB-0xF - reserved (Receive Array only)
11313  *
11314  *
11315  * This routine assumes that the value has already been sanity checked.
11316  */
11317 static u32 encoded_size(u32 size)
11318 {
11319         switch (size) {
11320         case   4 * 1024: return 0x1;
11321         case   8 * 1024: return 0x2;
11322         case  16 * 1024: return 0x3;
11323         case  32 * 1024: return 0x4;
11324         case  64 * 1024: return 0x5;
11325         case 128 * 1024: return 0x6;
11326         case 256 * 1024: return 0x7;
11327         case 512 * 1024: return 0x8;
11328         case   1 * 1024 * 1024: return 0x9;
11329         case   2 * 1024 * 1024: return 0xa;
11330         }
11331         return 0x1;     /* if invalid, go with the minimum size */
11332 }
11333
11334 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
11335 {
11336         struct hfi1_ctxtdata *rcd;
11337         u64 rcvctrl, reg;
11338         int did_enable = 0;
11339
11340         rcd = dd->rcd[ctxt];
11341         if (!rcd)
11342                 return;
11343
11344         hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
11345
11346         rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
11347         /* if the context already enabled, don't do the extra steps */
11348         if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
11349             !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
11350                 /* reset the tail and hdr addresses, and sequence count */
11351                 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
11352                                 rcd->rcvhdrq_phys);
11353                 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
11354                         write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11355                                         rcd->rcvhdrqtailaddr_phys);
11356                 rcd->seq_cnt = 1;
11357
11358                 /* reset the cached receive header queue head value */
11359                 rcd->head = 0;
11360
11361                 /*
11362                  * Zero the receive header queue so we don't get false
11363                  * positives when checking the sequence number.  The
11364                  * sequence numbers could land exactly on the same spot.
11365                  * E.g. a rcd restart before the receive header wrapped.
11366                  */
11367                 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
11368
11369                 /* starting timeout */
11370                 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
11371
11372                 /* enable the context */
11373                 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
11374
11375                 /* clean the egr buffer size first */
11376                 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11377                 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
11378                                 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
11379                                         << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
11380
11381                 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
11382                 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
11383                 did_enable = 1;
11384
11385                 /* zero RcvEgrIndexHead */
11386                 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
11387
11388                 /* set eager count and base index */
11389                 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
11390                         & RCV_EGR_CTRL_EGR_CNT_MASK)
11391                        << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
11392                         (((rcd->eager_base >> RCV_SHIFT)
11393                           & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
11394                          << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
11395                 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
11396
11397                 /*
11398                  * Set TID (expected) count and base index.
11399                  * rcd->expected_count is set to individual RcvArray entries,
11400                  * not pairs, and the CSR takes a pair-count in groups of
11401                  * four, so divide by 8.
11402                  */
11403                 reg = (((rcd->expected_count >> RCV_SHIFT)
11404                                         & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
11405                                 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
11406                       (((rcd->expected_base >> RCV_SHIFT)
11407                                         & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
11408                                 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
11409                 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
11410                 if (ctxt == HFI1_CTRL_CTXT)
11411                         write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
11412         }
11413         if (op & HFI1_RCVCTRL_CTXT_DIS) {
11414                 write_csr(dd, RCV_VL15, 0);
11415                 /*
11416                  * When receive context is being disabled turn on tail
11417                  * update with a dummy tail address and then disable
11418                  * receive context.
11419                  */
11420                 if (dd->rcvhdrtail_dummy_physaddr) {
11421                         write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11422                                         dd->rcvhdrtail_dummy_physaddr);
11423                         /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
11424                         rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11425                 }
11426
11427                 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
11428         }
11429         if (op & HFI1_RCVCTRL_INTRAVAIL_ENB)
11430                 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11431         if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
11432                 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11433         if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys)
11434                 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11435         if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
11436                 /* See comment on RcvCtxtCtrl.TailUpd above */
11437                 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
11438                         rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11439         }
11440         if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
11441                 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11442         if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
11443                 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11444         if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
11445                 /*
11446                  * In one-packet-per-eager mode, the size comes from
11447                  * the RcvArray entry.
11448                  */
11449                 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11450                 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11451         }
11452         if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
11453                 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11454         if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
11455                 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11456         if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
11457                 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11458         if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
11459                 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11460         if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
11461                 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11462         rcd->rcvctrl = rcvctrl;
11463         hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
11464         write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl);
11465
11466         /* work around sticky RcvCtxtStatus.BlockedRHQFull */
11467         if (did_enable &&
11468             (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
11469                 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11470                 if (reg != 0) {
11471                         dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
11472                                     ctxt, reg);
11473                         read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11474                         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
11475                         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
11476                         read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11477                         reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11478                         dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
11479                                     ctxt, reg, reg == 0 ? "not" : "still");
11480                 }
11481         }
11482
11483         if (did_enable) {
11484                 /*
11485                  * The interrupt timeout and count must be set after
11486                  * the context is enabled to take effect.
11487                  */
11488                 /* set interrupt timeout */
11489                 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
11490                                 (u64)rcd->rcvavail_timeout <<
11491                                 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11492
11493                 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
11494                 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
11495                 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11496         }
11497
11498         if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
11499                 /*
11500                  * If the context has been disabled and the Tail Update has
11501                  * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
11502                  * so it doesn't contain an address that is invalid.
11503                  */
11504                 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11505                                 dd->rcvhdrtail_dummy_physaddr);
11506 }
11507
11508 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
11509 {
11510         int ret;
11511         u64 val = 0;
11512
11513         if (namep) {
11514                 ret = dd->cntrnameslen;
11515                 *namep = dd->cntrnames;
11516         } else {
11517                 const struct cntr_entry *entry;
11518                 int i, j;
11519
11520                 ret = (dd->ndevcntrs) * sizeof(u64);
11521
11522                 /* Get the start of the block of counters */
11523                 *cntrp = dd->cntrs;
11524
11525                 /*
11526                  * Now go and fill in each counter in the block.
11527                  */
11528                 for (i = 0; i < DEV_CNTR_LAST; i++) {
11529                         entry = &dev_cntrs[i];
11530                         hfi1_cdbg(CNTR, "reading %s", entry->name);
11531                         if (entry->flags & CNTR_DISABLED) {
11532                                 /* Nothing */
11533                                 hfi1_cdbg(CNTR, "\tDisabled\n");
11534                         } else {
11535                                 if (entry->flags & CNTR_VL) {
11536                                         hfi1_cdbg(CNTR, "\tPer VL\n");
11537                                         for (j = 0; j < C_VL_COUNT; j++) {
11538                                                 val = entry->rw_cntr(entry,
11539                                                                   dd, j,
11540                                                                   CNTR_MODE_R,
11541                                                                   0);
11542                                                 hfi1_cdbg(
11543                                                    CNTR,
11544                                                    "\t\tRead 0x%llx for %d\n",
11545                                                    val, j);
11546                                                 dd->cntrs[entry->offset + j] =
11547                                                                             val;
11548                                         }
11549                                 } else if (entry->flags & CNTR_SDMA) {
11550                                         hfi1_cdbg(CNTR,
11551                                                   "\t Per SDMA Engine\n");
11552                                         for (j = 0; j < dd->chip_sdma_engines;
11553                                              j++) {
11554                                                 val =
11555                                                 entry->rw_cntr(entry, dd, j,
11556                                                                CNTR_MODE_R, 0);
11557                                                 hfi1_cdbg(CNTR,
11558                                                           "\t\tRead 0x%llx for %d\n",
11559                                                           val, j);
11560                                                 dd->cntrs[entry->offset + j] =
11561                                                                         val;
11562                                         }
11563                                 } else {
11564                                         val = entry->rw_cntr(entry, dd,
11565                                                         CNTR_INVALID_VL,
11566                                                         CNTR_MODE_R, 0);
11567                                         dd->cntrs[entry->offset] = val;
11568                                         hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11569                                 }
11570                         }
11571                 }
11572         }
11573         return ret;
11574 }
11575
11576 /*
11577  * Used by sysfs to create files for hfi stats to read
11578  */
11579 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
11580 {
11581         int ret;
11582         u64 val = 0;
11583
11584         if (namep) {
11585                 ret = ppd->dd->portcntrnameslen;
11586                 *namep = ppd->dd->portcntrnames;
11587         } else {
11588                 const struct cntr_entry *entry;
11589                 int i, j;
11590
11591                 ret = ppd->dd->nportcntrs * sizeof(u64);
11592                 *cntrp = ppd->cntrs;
11593
11594                 for (i = 0; i < PORT_CNTR_LAST; i++) {
11595                         entry = &port_cntrs[i];
11596                         hfi1_cdbg(CNTR, "reading %s", entry->name);
11597                         if (entry->flags & CNTR_DISABLED) {
11598                                 /* Nothing */
11599                                 hfi1_cdbg(CNTR, "\tDisabled\n");
11600                                 continue;
11601                         }
11602
11603                         if (entry->flags & CNTR_VL) {
11604                                 hfi1_cdbg(CNTR, "\tPer VL");
11605                                 for (j = 0; j < C_VL_COUNT; j++) {
11606                                         val = entry->rw_cntr(entry, ppd, j,
11607                                                                CNTR_MODE_R,
11608                                                                0);
11609                                         hfi1_cdbg(
11610                                            CNTR,
11611                                            "\t\tRead 0x%llx for %d",
11612                                            val, j);
11613                                         ppd->cntrs[entry->offset + j] = val;
11614                                 }
11615                         } else {
11616                                 val = entry->rw_cntr(entry, ppd,
11617                                                        CNTR_INVALID_VL,
11618                                                        CNTR_MODE_R,
11619                                                        0);
11620                                 ppd->cntrs[entry->offset] = val;
11621                                 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
11622                         }
11623                 }
11624         }
11625         return ret;
11626 }
11627
11628 static void free_cntrs(struct hfi1_devdata *dd)
11629 {
11630         struct hfi1_pportdata *ppd;
11631         int i;
11632
11633         if (dd->synth_stats_timer.data)
11634                 del_timer_sync(&dd->synth_stats_timer);
11635         dd->synth_stats_timer.data = 0;
11636         ppd = (struct hfi1_pportdata *)(dd + 1);
11637         for (i = 0; i < dd->num_pports; i++, ppd++) {
11638                 kfree(ppd->cntrs);
11639                 kfree(ppd->scntrs);
11640                 free_percpu(ppd->ibport_data.rvp.rc_acks);
11641                 free_percpu(ppd->ibport_data.rvp.rc_qacks);
11642                 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
11643                 ppd->cntrs = NULL;
11644                 ppd->scntrs = NULL;
11645                 ppd->ibport_data.rvp.rc_acks = NULL;
11646                 ppd->ibport_data.rvp.rc_qacks = NULL;
11647                 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
11648         }
11649         kfree(dd->portcntrnames);
11650         dd->portcntrnames = NULL;
11651         kfree(dd->cntrs);
11652         dd->cntrs = NULL;
11653         kfree(dd->scntrs);
11654         dd->scntrs = NULL;
11655         kfree(dd->cntrnames);
11656         dd->cntrnames = NULL;
11657 }
11658
11659 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
11660 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF
11661
11662 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
11663                               u64 *psval, void *context, int vl)
11664 {
11665         u64 val;
11666         u64 sval = *psval;
11667
11668         if (entry->flags & CNTR_DISABLED) {
11669                 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11670                 return 0;
11671         }
11672
11673         hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11674
11675         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
11676
11677         /* If its a synthetic counter there is more work we need to do */
11678         if (entry->flags & CNTR_SYNTH) {
11679                 if (sval == CNTR_MAX) {
11680                         /* No need to read already saturated */
11681                         return CNTR_MAX;
11682                 }
11683
11684                 if (entry->flags & CNTR_32BIT) {
11685                         /* 32bit counters can wrap multiple times */
11686                         u64 upper = sval >> 32;
11687                         u64 lower = (sval << 32) >> 32;
11688
11689                         if (lower > val) { /* hw wrapped */
11690                                 if (upper == CNTR_32BIT_MAX)
11691                                         val = CNTR_MAX;
11692                                 else
11693                                         upper++;
11694                         }
11695
11696                         if (val != CNTR_MAX)
11697                                 val = (upper << 32) | val;
11698
11699                 } else {
11700                         /* If we rolled we are saturated */
11701                         if ((val < sval) || (val > CNTR_MAX))
11702                                 val = CNTR_MAX;
11703                 }
11704         }
11705
11706         *psval = val;
11707
11708         hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11709
11710         return val;
11711 }
11712
11713 static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
11714                                struct cntr_entry *entry,
11715                                u64 *psval, void *context, int vl, u64 data)
11716 {
11717         u64 val;
11718
11719         if (entry->flags & CNTR_DISABLED) {
11720                 dd_dev_err(dd, "Counter %s not enabled", entry->name);
11721                 return 0;
11722         }
11723
11724         hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
11725
11726         if (entry->flags & CNTR_SYNTH) {
11727                 *psval = data;
11728                 if (entry->flags & CNTR_32BIT) {
11729                         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11730                                              (data << 32) >> 32);
11731                         val = data; /* return the full 64bit value */
11732                 } else {
11733                         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
11734                                              data);
11735                 }
11736         } else {
11737                 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
11738         }
11739
11740         *psval = val;
11741
11742         hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
11743
11744         return val;
11745 }
11746
11747 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
11748 {
11749         struct cntr_entry *entry;
11750         u64 *sval;
11751
11752         entry = &dev_cntrs[index];
11753         sval = dd->scntrs + entry->offset;
11754
11755         if (vl != CNTR_INVALID_VL)
11756                 sval += vl;
11757
11758         return read_dev_port_cntr(dd, entry, sval, dd, vl);
11759 }
11760
11761 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
11762 {
11763         struct cntr_entry *entry;
11764         u64 *sval;
11765
11766         entry = &dev_cntrs[index];
11767         sval = dd->scntrs + entry->offset;
11768
11769         if (vl != CNTR_INVALID_VL)
11770                 sval += vl;
11771
11772         return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
11773 }
11774
11775 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
11776 {
11777         struct cntr_entry *entry;
11778         u64 *sval;
11779
11780         entry = &port_cntrs[index];
11781         sval = ppd->scntrs + entry->offset;
11782
11783         if (vl != CNTR_INVALID_VL)
11784                 sval += vl;
11785
11786         if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11787             (index <= C_RCV_HDR_OVF_LAST)) {
11788                 /* We do not want to bother for disabled contexts */
11789                 return 0;
11790         }
11791
11792         return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
11793 }
11794
11795 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
11796 {
11797         struct cntr_entry *entry;
11798         u64 *sval;
11799
11800         entry = &port_cntrs[index];
11801         sval = ppd->scntrs + entry->offset;
11802
11803         if (vl != CNTR_INVALID_VL)
11804                 sval += vl;
11805
11806         if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
11807             (index <= C_RCV_HDR_OVF_LAST)) {
11808                 /* We do not want to bother for disabled contexts */
11809                 return 0;
11810         }
11811
11812         return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
11813 }
11814
11815 static void update_synth_timer(unsigned long opaque)
11816 {
11817         u64 cur_tx;
11818         u64 cur_rx;
11819         u64 total_flits;
11820         u8 update = 0;
11821         int i, j, vl;
11822         struct hfi1_pportdata *ppd;
11823         struct cntr_entry *entry;
11824
11825         struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque;
11826
11827         /*
11828          * Rather than keep beating on the CSRs pick a minimal set that we can
11829          * check to watch for potential roll over. We can do this by looking at
11830          * the number of flits sent/recv. If the total flits exceeds 32bits then
11831          * we have to iterate all the counters and update.
11832          */
11833         entry = &dev_cntrs[C_DC_RCV_FLITS];
11834         cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
11835
11836         entry = &dev_cntrs[C_DC_XMIT_FLITS];
11837         cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
11838
11839         hfi1_cdbg(
11840             CNTR,
11841             "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
11842             dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
11843
11844         if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
11845                 /*
11846                  * May not be strictly necessary to update but it won't hurt and
11847                  * simplifies the logic here.
11848                  */
11849                 update = 1;
11850                 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
11851                           dd->unit);
11852         } else {
11853                 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
11854                 hfi1_cdbg(CNTR,
11855                           "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
11856                           total_flits, (u64)CNTR_32BIT_MAX);
11857                 if (total_flits >= CNTR_32BIT_MAX) {
11858                         hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
11859                                   dd->unit);
11860                         update = 1;
11861                 }
11862         }
11863
11864         if (update) {
11865                 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
11866                 for (i = 0; i < DEV_CNTR_LAST; i++) {
11867                         entry = &dev_cntrs[i];
11868                         if (entry->flags & CNTR_VL) {
11869                                 for (vl = 0; vl < C_VL_COUNT; vl++)
11870                                         read_dev_cntr(dd, i, vl);
11871                         } else {
11872                                 read_dev_cntr(dd, i, CNTR_INVALID_VL);
11873                         }
11874                 }
11875                 ppd = (struct hfi1_pportdata *)(dd + 1);
11876                 for (i = 0; i < dd->num_pports; i++, ppd++) {
11877                         for (j = 0; j < PORT_CNTR_LAST; j++) {
11878                                 entry = &port_cntrs[j];
11879                                 if (entry->flags & CNTR_VL) {
11880                                         for (vl = 0; vl < C_VL_COUNT; vl++)
11881                                                 read_port_cntr(ppd, j, vl);
11882                                 } else {
11883                                         read_port_cntr(ppd, j, CNTR_INVALID_VL);
11884                                 }
11885                         }
11886                 }
11887
11888                 /*
11889                  * We want the value in the register. The goal is to keep track
11890                  * of the number of "ticks" not the counter value. In other
11891                  * words if the register rolls we want to notice it and go ahead
11892                  * and force an update.
11893                  */
11894                 entry = &dev_cntrs[C_DC_XMIT_FLITS];
11895                 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
11896                                                 CNTR_MODE_R, 0);
11897
11898                 entry = &dev_cntrs[C_DC_RCV_FLITS];
11899                 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
11900                                                 CNTR_MODE_R, 0);
11901
11902                 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
11903                           dd->unit, dd->last_tx, dd->last_rx);
11904
11905         } else {
11906                 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
11907         }
11908
11909         mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
11910 }
11911
11912 #define C_MAX_NAME 13 /* 12 chars + one for /0 */
11913 static int init_cntrs(struct hfi1_devdata *dd)
11914 {
11915         int i, rcv_ctxts, j;
11916         size_t sz;
11917         char *p;
11918         char name[C_MAX_NAME];
11919         struct hfi1_pportdata *ppd;
11920         const char *bit_type_32 = ",32";
11921         const int bit_type_32_sz = strlen(bit_type_32);
11922
11923         /* set up the stats timer; the add_timer is done at the end */
11924         setup_timer(&dd->synth_stats_timer, update_synth_timer,
11925                     (unsigned long)dd);
11926
11927         /***********************/
11928         /* per device counters */
11929         /***********************/
11930
11931         /* size names and determine how many we have*/
11932         dd->ndevcntrs = 0;
11933         sz = 0;
11934
11935         for (i = 0; i < DEV_CNTR_LAST; i++) {
11936                 if (dev_cntrs[i].flags & CNTR_DISABLED) {
11937                         hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
11938                         continue;
11939                 }
11940
11941                 if (dev_cntrs[i].flags & CNTR_VL) {
11942                         dev_cntrs[i].offset = dd->ndevcntrs;
11943                         for (j = 0; j < C_VL_COUNT; j++) {
11944                                 snprintf(name, C_MAX_NAME, "%s%d",
11945                                          dev_cntrs[i].name, vl_from_idx(j));
11946                                 sz += strlen(name);
11947                                 /* Add ",32" for 32-bit counters */
11948                                 if (dev_cntrs[i].flags & CNTR_32BIT)
11949                                         sz += bit_type_32_sz;
11950                                 sz++;
11951                                 dd->ndevcntrs++;
11952                         }
11953                 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
11954                         dev_cntrs[i].offset = dd->ndevcntrs;
11955                         for (j = 0; j < dd->chip_sdma_engines; j++) {
11956                                 snprintf(name, C_MAX_NAME, "%s%d",
11957                                          dev_cntrs[i].name, j);
11958                                 sz += strlen(name);
11959                                 /* Add ",32" for 32-bit counters */
11960                                 if (dev_cntrs[i].flags & CNTR_32BIT)
11961                                         sz += bit_type_32_sz;
11962                                 sz++;
11963                                 dd->ndevcntrs++;
11964                         }
11965                 } else {
11966                         /* +1 for newline. */
11967                         sz += strlen(dev_cntrs[i].name) + 1;
11968                         /* Add ",32" for 32-bit counters */
11969                         if (dev_cntrs[i].flags & CNTR_32BIT)
11970                                 sz += bit_type_32_sz;
11971                         dev_cntrs[i].offset = dd->ndevcntrs;
11972                         dd->ndevcntrs++;
11973                 }
11974         }
11975
11976         /* allocate space for the counter values */
11977         dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
11978         if (!dd->cntrs)
11979                 goto bail;
11980
11981         dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
11982         if (!dd->scntrs)
11983                 goto bail;
11984
11985         /* allocate space for the counter names */
11986         dd->cntrnameslen = sz;
11987         dd->cntrnames = kmalloc(sz, GFP_KERNEL);
11988         if (!dd->cntrnames)
11989                 goto bail;
11990
11991         /* fill in the names */
11992         for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
11993                 if (dev_cntrs[i].flags & CNTR_DISABLED) {
11994                         /* Nothing */
11995                 } else if (dev_cntrs[i].flags & CNTR_VL) {
11996                         for (j = 0; j < C_VL_COUNT; j++) {
11997                                 snprintf(name, C_MAX_NAME, "%s%d",
11998                                          dev_cntrs[i].name,
11999                                          vl_from_idx(j));
12000                                 memcpy(p, name, strlen(name));
12001                                 p += strlen(name);
12002
12003                                 /* Counter is 32 bits */
12004                                 if (dev_cntrs[i].flags & CNTR_32BIT) {
12005                                         memcpy(p, bit_type_32, bit_type_32_sz);
12006                                         p += bit_type_32_sz;
12007                                 }
12008
12009                                 *p++ = '\n';
12010                         }
12011                 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12012                         for (j = 0; j < dd->chip_sdma_engines; j++) {
12013                                 snprintf(name, C_MAX_NAME, "%s%d",
12014                                          dev_cntrs[i].name, j);
12015                                 memcpy(p, name, strlen(name));
12016                                 p += strlen(name);
12017
12018                                 /* Counter is 32 bits */
12019                                 if (dev_cntrs[i].flags & CNTR_32BIT) {
12020                                         memcpy(p, bit_type_32, bit_type_32_sz);
12021                                         p += bit_type_32_sz;
12022                                 }
12023
12024                                 *p++ = '\n';
12025                         }
12026                 } else {
12027                         memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12028                         p += strlen(dev_cntrs[i].name);
12029
12030                         /* Counter is 32 bits */
12031                         if (dev_cntrs[i].flags & CNTR_32BIT) {
12032                                 memcpy(p, bit_type_32, bit_type_32_sz);
12033                                 p += bit_type_32_sz;
12034                         }
12035
12036                         *p++ = '\n';
12037                 }
12038         }
12039
12040         /*********************/
12041         /* per port counters */
12042         /*********************/
12043
12044         /*
12045          * Go through the counters for the overflows and disable the ones we
12046          * don't need. This varies based on platform so we need to do it
12047          * dynamically here.
12048          */
12049         rcv_ctxts = dd->num_rcv_contexts;
12050         for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12051              i <= C_RCV_HDR_OVF_LAST; i++) {
12052                 port_cntrs[i].flags |= CNTR_DISABLED;
12053         }
12054
12055         /* size port counter names and determine how many we have*/
12056         sz = 0;
12057         dd->nportcntrs = 0;
12058         for (i = 0; i < PORT_CNTR_LAST; i++) {
12059                 if (port_cntrs[i].flags & CNTR_DISABLED) {
12060                         hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12061                         continue;
12062                 }
12063
12064                 if (port_cntrs[i].flags & CNTR_VL) {
12065                         port_cntrs[i].offset = dd->nportcntrs;
12066                         for (j = 0; j < C_VL_COUNT; j++) {
12067                                 snprintf(name, C_MAX_NAME, "%s%d",
12068                                          port_cntrs[i].name, vl_from_idx(j));
12069                                 sz += strlen(name);
12070                                 /* Add ",32" for 32-bit counters */
12071                                 if (port_cntrs[i].flags & CNTR_32BIT)
12072                                         sz += bit_type_32_sz;
12073                                 sz++;
12074                                 dd->nportcntrs++;
12075                         }
12076                 } else {
12077                         /* +1 for newline */
12078                         sz += strlen(port_cntrs[i].name) + 1;
12079                         /* Add ",32" for 32-bit counters */
12080                         if (port_cntrs[i].flags & CNTR_32BIT)
12081                                 sz += bit_type_32_sz;
12082                         port_cntrs[i].offset = dd->nportcntrs;
12083                         dd->nportcntrs++;
12084                 }
12085         }
12086
12087         /* allocate space for the counter names */
12088         dd->portcntrnameslen = sz;
12089         dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12090         if (!dd->portcntrnames)
12091                 goto bail;
12092
12093         /* fill in port cntr names */
12094         for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12095                 if (port_cntrs[i].flags & CNTR_DISABLED)
12096                         continue;
12097
12098                 if (port_cntrs[i].flags & CNTR_VL) {
12099                         for (j = 0; j < C_VL_COUNT; j++) {
12100                                 snprintf(name, C_MAX_NAME, "%s%d",
12101                                          port_cntrs[i].name, vl_from_idx(j));
12102                                 memcpy(p, name, strlen(name));
12103                                 p += strlen(name);
12104
12105                                 /* Counter is 32 bits */
12106                                 if (port_cntrs[i].flags & CNTR_32BIT) {
12107                                         memcpy(p, bit_type_32, bit_type_32_sz);
12108                                         p += bit_type_32_sz;
12109                                 }
12110
12111                                 *p++ = '\n';
12112                         }
12113                 } else {
12114                         memcpy(p, port_cntrs[i].name,
12115                                strlen(port_cntrs[i].name));
12116                         p += strlen(port_cntrs[i].name);
12117
12118                         /* Counter is 32 bits */
12119                         if (port_cntrs[i].flags & CNTR_32BIT) {
12120                                 memcpy(p, bit_type_32, bit_type_32_sz);
12121                                 p += bit_type_32_sz;
12122                         }
12123
12124                         *p++ = '\n';
12125                 }
12126         }
12127
12128         /* allocate per port storage for counter values */
12129         ppd = (struct hfi1_pportdata *)(dd + 1);
12130         for (i = 0; i < dd->num_pports; i++, ppd++) {
12131                 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12132                 if (!ppd->cntrs)
12133                         goto bail;
12134
12135                 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12136                 if (!ppd->scntrs)
12137                         goto bail;
12138         }
12139
12140         /* CPU counters need to be allocated and zeroed */
12141         if (init_cpu_counters(dd))
12142                 goto bail;
12143
12144         mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12145         return 0;
12146 bail:
12147         free_cntrs(dd);
12148         return -ENOMEM;
12149 }
12150
12151 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12152 {
12153         switch (chip_lstate) {
12154         default:
12155                 dd_dev_err(dd,
12156                            "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12157                            chip_lstate);
12158                 /* fall through */
12159         case LSTATE_DOWN:
12160                 return IB_PORT_DOWN;
12161         case LSTATE_INIT:
12162                 return IB_PORT_INIT;
12163         case LSTATE_ARMED:
12164                 return IB_PORT_ARMED;
12165         case LSTATE_ACTIVE:
12166                 return IB_PORT_ACTIVE;
12167         }
12168 }
12169
12170 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12171 {
12172         /* look at the HFI meta-states only */
12173         switch (chip_pstate & 0xf0) {
12174         default:
12175                 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
12176                            chip_pstate);
12177                 /* fall through */
12178         case PLS_DISABLED:
12179                 return IB_PORTPHYSSTATE_DISABLED;
12180         case PLS_OFFLINE:
12181                 return OPA_PORTPHYSSTATE_OFFLINE;
12182         case PLS_POLLING:
12183                 return IB_PORTPHYSSTATE_POLLING;
12184         case PLS_CONFIGPHY:
12185                 return IB_PORTPHYSSTATE_TRAINING;
12186         case PLS_LINKUP:
12187                 return IB_PORTPHYSSTATE_LINKUP;
12188         case PLS_PHYTEST:
12189                 return IB_PORTPHYSSTATE_PHY_TEST;
12190         }
12191 }
12192
12193 /* return the OPA port logical state name */
12194 const char *opa_lstate_name(u32 lstate)
12195 {
12196         static const char * const port_logical_names[] = {
12197                 "PORT_NOP",
12198                 "PORT_DOWN",
12199                 "PORT_INIT",
12200                 "PORT_ARMED",
12201                 "PORT_ACTIVE",
12202                 "PORT_ACTIVE_DEFER",
12203         };
12204         if (lstate < ARRAY_SIZE(port_logical_names))
12205                 return port_logical_names[lstate];
12206         return "unknown";
12207 }
12208
12209 /* return the OPA port physical state name */
12210 const char *opa_pstate_name(u32 pstate)
12211 {
12212         static const char * const port_physical_names[] = {
12213                 "PHYS_NOP",
12214                 "reserved1",
12215                 "PHYS_POLL",
12216                 "PHYS_DISABLED",
12217                 "PHYS_TRAINING",
12218                 "PHYS_LINKUP",
12219                 "PHYS_LINK_ERR_RECOVER",
12220                 "PHYS_PHY_TEST",
12221                 "reserved8",
12222                 "PHYS_OFFLINE",
12223                 "PHYS_GANGED",
12224                 "PHYS_TEST",
12225         };
12226         if (pstate < ARRAY_SIZE(port_physical_names))
12227                 return port_physical_names[pstate];
12228         return "unknown";
12229 }
12230
12231 /*
12232  * Read the hardware link state and set the driver's cached value of it.
12233  * Return the (new) current value.
12234  */
12235 u32 get_logical_state(struct hfi1_pportdata *ppd)
12236 {
12237         u32 new_state;
12238
12239         new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
12240         if (new_state != ppd->lstate) {
12241                 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
12242                             opa_lstate_name(new_state), new_state);
12243                 ppd->lstate = new_state;
12244         }
12245         /*
12246          * Set port status flags in the page mapped into userspace
12247          * memory. Do it here to ensure a reliable state - this is
12248          * the only function called by all state handling code.
12249          * Always set the flags due to the fact that the cache value
12250          * might have been changed explicitly outside of this
12251          * function.
12252          */
12253         if (ppd->statusp) {
12254                 switch (ppd->lstate) {
12255                 case IB_PORT_DOWN:
12256                 case IB_PORT_INIT:
12257                         *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12258                                            HFI1_STATUS_IB_READY);
12259                         break;
12260                 case IB_PORT_ARMED:
12261                         *ppd->statusp |= HFI1_STATUS_IB_CONF;
12262                         break;
12263                 case IB_PORT_ACTIVE:
12264                         *ppd->statusp |= HFI1_STATUS_IB_READY;
12265                         break;
12266                 }
12267         }
12268         return ppd->lstate;
12269 }
12270
12271 /**
12272  * wait_logical_linkstate - wait for an IB link state change to occur
12273  * @ppd: port device
12274  * @state: the state to wait for
12275  * @msecs: the number of milliseconds to wait
12276  *
12277  * Wait up to msecs milliseconds for IB link state change to occur.
12278  * For now, take the easy polling route.
12279  * Returns 0 if state reached, otherwise -ETIMEDOUT.
12280  */
12281 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12282                                   int msecs)
12283 {
12284         unsigned long timeout;
12285
12286         timeout = jiffies + msecs_to_jiffies(msecs);
12287         while (1) {
12288                 if (get_logical_state(ppd) == state)
12289                         return 0;
12290                 if (time_after(jiffies, timeout))
12291                         break;
12292                 msleep(20);
12293         }
12294         dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
12295
12296         return -ETIMEDOUT;
12297 }
12298
12299 u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
12300 {
12301         u32 pstate;
12302         u32 ib_pstate;
12303
12304         pstate = read_physical_state(ppd->dd);
12305         ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
12306         if (ppd->last_pstate != ib_pstate) {
12307                 dd_dev_info(ppd->dd,
12308                             "%s: physical state changed to %s (0x%x), phy 0x%x\n",
12309                             __func__, opa_pstate_name(ib_pstate), ib_pstate,
12310                             pstate);
12311                 ppd->last_pstate = ib_pstate;
12312         }
12313         return ib_pstate;
12314 }
12315
12316 /*
12317  * Read/modify/write ASIC_QSFP register bits as selected by mask
12318  * data: 0 or 1 in the positions depending on what needs to be written
12319  * dir: 0 for read, 1 for write
12320  * mask: select by setting
12321  *      I2CCLK  (bit 0)
12322  *      I2CDATA (bit 1)
12323  */
12324 u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir,
12325                   u32 mask)
12326 {
12327         u64 qsfp_oe, target_oe;
12328
12329         target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE;
12330         if (mask) {
12331                 /* We are writing register bits, so lock access */
12332                 dir &= mask;
12333                 data &= mask;
12334
12335                 qsfp_oe = read_csr(dd, target_oe);
12336                 qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir;
12337                 write_csr(dd, target_oe, qsfp_oe);
12338         }
12339         /* We are exclusively reading bits here, but it is unlikely
12340          * we'll get valid data when we set the direction of the pin
12341          * in the same call, so read should call this function again
12342          * to get valid data
12343          */
12344         return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN);
12345 }
12346
12347 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12348 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12349
12350 #define SET_STATIC_RATE_CONTROL_SMASK(r) \
12351 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12352
12353 int hfi1_init_ctxt(struct send_context *sc)
12354 {
12355         if (sc) {
12356                 struct hfi1_devdata *dd = sc->dd;
12357                 u64 reg;
12358                 u8 set = (sc->type == SC_USER ?
12359                           HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
12360                           HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
12361                 reg = read_kctxt_csr(dd, sc->hw_context,
12362                                      SEND_CTXT_CHECK_ENABLE);
12363                 if (set)
12364                         CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
12365                 else
12366                         SET_STATIC_RATE_CONTROL_SMASK(reg);
12367                 write_kctxt_csr(dd, sc->hw_context,
12368                                 SEND_CTXT_CHECK_ENABLE, reg);
12369         }
12370         return 0;
12371 }
12372
12373 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
12374 {
12375         int ret = 0;
12376         u64 reg;
12377
12378         if (dd->icode != ICODE_RTL_SILICON) {
12379                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
12380                         dd_dev_info(dd, "%s: tempsense not supported by HW\n",
12381                                     __func__);
12382                 return -EINVAL;
12383         }
12384         reg = read_csr(dd, ASIC_STS_THERM);
12385         temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
12386                       ASIC_STS_THERM_CURR_TEMP_MASK);
12387         temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
12388                         ASIC_STS_THERM_LO_TEMP_MASK);
12389         temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
12390                         ASIC_STS_THERM_HI_TEMP_MASK);
12391         temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
12392                           ASIC_STS_THERM_CRIT_TEMP_MASK);
12393         /* triggers is a 3-bit value - 1 bit per trigger. */
12394         temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
12395
12396         return ret;
12397 }
12398
12399 /* ========================================================================= */
12400
12401 /*
12402  * Enable/disable chip from delivering interrupts.
12403  */
12404 void set_intr_state(struct hfi1_devdata *dd, u32 enable)
12405 {
12406         int i;
12407
12408         /*
12409          * In HFI, the mask needs to be 1 to allow interrupts.
12410          */
12411         if (enable) {
12412                 /* enable all interrupts */
12413                 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12414                         write_csr(dd, CCE_INT_MASK + (8 * i), ~(u64)0);
12415
12416                 init_qsfp_int(dd);
12417         } else {
12418                 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12419                         write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
12420         }
12421 }
12422
12423 /*
12424  * Clear all interrupt sources on the chip.
12425  */
12426 static void clear_all_interrupts(struct hfi1_devdata *dd)
12427 {
12428         int i;
12429
12430         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12431                 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
12432
12433         write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
12434         write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
12435         write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
12436         write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
12437         write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
12438         write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
12439         write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
12440         for (i = 0; i < dd->chip_send_contexts; i++)
12441                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
12442         for (i = 0; i < dd->chip_sdma_engines; i++)
12443                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
12444
12445         write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
12446         write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
12447         write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
12448 }
12449
12450 /* Move to pcie.c? */
12451 static void disable_intx(struct pci_dev *pdev)
12452 {
12453         pci_intx(pdev, 0);
12454 }
12455
12456 static void clean_up_interrupts(struct hfi1_devdata *dd)
12457 {
12458         int i;
12459
12460         /* remove irqs - must happen before disabling/turning off */
12461         if (dd->num_msix_entries) {
12462                 /* MSI-X */
12463                 struct hfi1_msix_entry *me = dd->msix_entries;
12464
12465                 for (i = 0; i < dd->num_msix_entries; i++, me++) {
12466                         if (!me->arg) /* => no irq, no affinity */
12467                                 continue;
12468                         hfi1_put_irq_affinity(dd, &dd->msix_entries[i]);
12469                         free_irq(me->msix.vector, me->arg);
12470                 }
12471         } else {
12472                 /* INTx */
12473                 if (dd->requested_intx_irq) {
12474                         free_irq(dd->pcidev->irq, dd);
12475                         dd->requested_intx_irq = 0;
12476                 }
12477         }
12478
12479         /* turn off interrupts */
12480         if (dd->num_msix_entries) {
12481                 /* MSI-X */
12482                 pci_disable_msix(dd->pcidev);
12483         } else {
12484                 /* INTx */
12485                 disable_intx(dd->pcidev);
12486         }
12487
12488         /* clean structures */
12489         kfree(dd->msix_entries);
12490         dd->msix_entries = NULL;
12491         dd->num_msix_entries = 0;
12492 }
12493
12494 /*
12495  * Remap the interrupt source from the general handler to the given MSI-X
12496  * interrupt.
12497  */
12498 static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
12499 {
12500         u64 reg;
12501         int m, n;
12502
12503         /* clear from the handled mask of the general interrupt */
12504         m = isrc / 64;
12505         n = isrc % 64;
12506         dd->gi_mask[m] &= ~((u64)1 << n);
12507
12508         /* direct the chip source to the given MSI-X interrupt */
12509         m = isrc / 8;
12510         n = isrc % 8;
12511         reg = read_csr(dd, CCE_INT_MAP + (8 * m));
12512         reg &= ~((u64)0xff << (8 * n));
12513         reg |= ((u64)msix_intr & 0xff) << (8 * n);
12514         write_csr(dd, CCE_INT_MAP + (8 * m), reg);
12515 }
12516
12517 static void remap_sdma_interrupts(struct hfi1_devdata *dd,
12518                                   int engine, int msix_intr)
12519 {
12520         /*
12521          * SDMA engine interrupt sources grouped by type, rather than
12522          * engine.  Per-engine interrupts are as follows:
12523          *      SDMA
12524          *      SDMAProgress
12525          *      SDMAIdle
12526          */
12527         remap_intr(dd, IS_SDMA_START + 0 * TXE_NUM_SDMA_ENGINES + engine,
12528                    msix_intr);
12529         remap_intr(dd, IS_SDMA_START + 1 * TXE_NUM_SDMA_ENGINES + engine,
12530                    msix_intr);
12531         remap_intr(dd, IS_SDMA_START + 2 * TXE_NUM_SDMA_ENGINES + engine,
12532                    msix_intr);
12533 }
12534
12535 static int request_intx_irq(struct hfi1_devdata *dd)
12536 {
12537         int ret;
12538
12539         snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d",
12540                  dd->unit);
12541         ret = request_irq(dd->pcidev->irq, general_interrupt,
12542                           IRQF_SHARED, dd->intx_name, dd);
12543         if (ret)
12544                 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n",
12545                            ret);
12546         else
12547                 dd->requested_intx_irq = 1;
12548         return ret;
12549 }
12550
12551 static int request_msix_irqs(struct hfi1_devdata *dd)
12552 {
12553         int first_general, last_general;
12554         int first_sdma, last_sdma;
12555         int first_rx, last_rx;
12556         int i, ret = 0;
12557
12558         /* calculate the ranges we are going to use */
12559         first_general = 0;
12560         last_general = first_general + 1;
12561         first_sdma = last_general;
12562         last_sdma = first_sdma + dd->num_sdma;
12563         first_rx = last_sdma;
12564         last_rx = first_rx + dd->n_krcv_queues;
12565
12566         /*
12567          * Sanity check - the code expects all SDMA chip source
12568          * interrupts to be in the same CSR, starting at bit 0.  Verify
12569          * that this is true by checking the bit location of the start.
12570          */
12571         BUILD_BUG_ON(IS_SDMA_START % 64);
12572
12573         for (i = 0; i < dd->num_msix_entries; i++) {
12574                 struct hfi1_msix_entry *me = &dd->msix_entries[i];
12575                 const char *err_info;
12576                 irq_handler_t handler;
12577                 irq_handler_t thread = NULL;
12578                 void *arg;
12579                 int idx;
12580                 struct hfi1_ctxtdata *rcd = NULL;
12581                 struct sdma_engine *sde = NULL;
12582
12583                 /* obtain the arguments to request_irq */
12584                 if (first_general <= i && i < last_general) {
12585                         idx = i - first_general;
12586                         handler = general_interrupt;
12587                         arg = dd;
12588                         snprintf(me->name, sizeof(me->name),
12589                                  DRIVER_NAME "_%d", dd->unit);
12590                         err_info = "general";
12591                         me->type = IRQ_GENERAL;
12592                 } else if (first_sdma <= i && i < last_sdma) {
12593                         idx = i - first_sdma;
12594                         sde = &dd->per_sdma[idx];
12595                         handler = sdma_interrupt;
12596                         arg = sde;
12597                         snprintf(me->name, sizeof(me->name),
12598                                  DRIVER_NAME "_%d sdma%d", dd->unit, idx);
12599                         err_info = "sdma";
12600                         remap_sdma_interrupts(dd, idx, i);
12601                         me->type = IRQ_SDMA;
12602                 } else if (first_rx <= i && i < last_rx) {
12603                         idx = i - first_rx;
12604                         rcd = dd->rcd[idx];
12605                         /* no interrupt if no rcd */
12606                         if (!rcd)
12607                                 continue;
12608                         /*
12609                          * Set the interrupt register and mask for this
12610                          * context's interrupt.
12611                          */
12612                         rcd->ireg = (IS_RCVAVAIL_START + idx) / 64;
12613                         rcd->imask = ((u64)1) <<
12614                                         ((IS_RCVAVAIL_START + idx) % 64);
12615                         handler = receive_context_interrupt;
12616                         thread = receive_context_thread;
12617                         arg = rcd;
12618                         snprintf(me->name, sizeof(me->name),
12619                                  DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
12620                         err_info = "receive context";
12621                         remap_intr(dd, IS_RCVAVAIL_START + idx, i);
12622                         me->type = IRQ_RCVCTXT;
12623                 } else {
12624                         /* not in our expected range - complain, then
12625                          * ignore it
12626                          */
12627                         dd_dev_err(dd,
12628                                    "Unexpected extra MSI-X interrupt %d\n", i);
12629                         continue;
12630                 }
12631                 /* no argument, no interrupt */
12632                 if (!arg)
12633                         continue;
12634                 /* make sure the name is terminated */
12635                 me->name[sizeof(me->name) - 1] = 0;
12636
12637                 ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
12638                                            me->name, arg);
12639                 if (ret) {
12640                         dd_dev_err(dd,
12641                                    "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
12642                                    err_info, me->msix.vector, idx, ret);
12643                         return ret;
12644                 }
12645                 /*
12646                  * assign arg after request_irq call, so it will be
12647                  * cleaned up
12648                  */
12649                 me->arg = arg;
12650
12651                 ret = hfi1_get_irq_affinity(dd, me);
12652                 if (ret)
12653                         dd_dev_err(dd,
12654                                    "unable to pin IRQ %d\n", ret);
12655         }
12656
12657         return ret;
12658 }
12659
12660 /*
12661  * Set the general handler to accept all interrupts, remap all
12662  * chip interrupts back to MSI-X 0.
12663  */
12664 static void reset_interrupts(struct hfi1_devdata *dd)
12665 {
12666         int i;
12667
12668         /* all interrupts handled by the general handler */
12669         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
12670                 dd->gi_mask[i] = ~(u64)0;
12671
12672         /* all chip interrupts map to MSI-X 0 */
12673         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
12674                 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
12675 }
12676
12677 static int set_up_interrupts(struct hfi1_devdata *dd)
12678 {
12679         struct hfi1_msix_entry *entries;
12680         u32 total, request;
12681         int i, ret;
12682         int single_interrupt = 0; /* we expect to have all the interrupts */
12683
12684         /*
12685          * Interrupt count:
12686          *      1 general, "slow path" interrupt (includes the SDMA engines
12687          *              slow source, SDMACleanupDone)
12688          *      N interrupts - one per used SDMA engine
12689          *      M interrupt - one per kernel receive context
12690          */
12691         total = 1 + dd->num_sdma + dd->n_krcv_queues;
12692
12693         entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
12694         if (!entries) {
12695                 ret = -ENOMEM;
12696                 goto fail;
12697         }
12698         /* 1-1 MSI-X entry assignment */
12699         for (i = 0; i < total; i++)
12700                 entries[i].msix.entry = i;
12701
12702         /* ask for MSI-X interrupts */
12703         request = total;
12704         request_msix(dd, &request, entries);
12705
12706         if (request == 0) {
12707                 /* using INTx */
12708                 /* dd->num_msix_entries already zero */
12709                 kfree(entries);
12710                 single_interrupt = 1;
12711                 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
12712         } else {
12713                 /* using MSI-X */
12714                 dd->num_msix_entries = request;
12715                 dd->msix_entries = entries;
12716
12717                 if (request != total) {
12718                         /* using MSI-X, with reduced interrupts */
12719                         dd_dev_err(
12720                                 dd,
12721                                 "cannot handle reduced interrupt case, want %u, got %u\n",
12722                                 total, request);
12723                         ret = -EINVAL;
12724                         goto fail;
12725                 }
12726                 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
12727         }
12728
12729         /* mask all interrupts */
12730         set_intr_state(dd, 0);
12731         /* clear all pending interrupts */
12732         clear_all_interrupts(dd);
12733
12734         /* reset general handler mask, chip MSI-X mappings */
12735         reset_interrupts(dd);
12736
12737         if (single_interrupt)
12738                 ret = request_intx_irq(dd);
12739         else
12740                 ret = request_msix_irqs(dd);
12741         if (ret)
12742                 goto fail;
12743
12744         return 0;
12745
12746 fail:
12747         clean_up_interrupts(dd);
12748         return ret;
12749 }
12750
12751 /*
12752  * Set up context values in dd.  Sets:
12753  *
12754  *      num_rcv_contexts - number of contexts being used
12755  *      n_krcv_queues - number of kernel contexts
12756  *      first_user_ctxt - first non-kernel context in array of contexts
12757  *      freectxts  - number of free user contexts
12758  *      num_send_contexts - number of PIO send contexts being used
12759  */
12760 static int set_up_context_variables(struct hfi1_devdata *dd)
12761 {
12762         int num_kernel_contexts;
12763         int total_contexts;
12764         int ret;
12765         unsigned ngroups;
12766         int qos_rmt_count;
12767         int user_rmt_reduced;
12768
12769         /*
12770          * Kernel receive contexts:
12771          * - min of 2 or 1 context/numa (excluding control context)
12772          * - Context 0 - control context (VL15/multicast/error)
12773          * - Context 1 - first kernel context
12774          * - Context 2 - second kernel context
12775          * ...
12776          */
12777         if (n_krcvqs)
12778                 /*
12779                  * n_krcvqs is the sum of module parameter kernel receive
12780                  * contexts, krcvqs[].  It does not include the control
12781                  * context, so add that.
12782                  */
12783                 num_kernel_contexts = n_krcvqs + 1;
12784         else
12785                 num_kernel_contexts = num_online_nodes() + 1;
12786         num_kernel_contexts =
12787                 max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts);
12788         /*
12789          * Every kernel receive context needs an ACK send context.
12790          * one send context is allocated for each VL{0-7} and VL15
12791          */
12792         if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
12793                 dd_dev_err(dd,
12794                            "Reducing # kernel rcv contexts to: %d, from %d\n",
12795                            (int)(dd->chip_send_contexts - num_vls - 1),
12796                            (int)num_kernel_contexts);
12797                 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
12798         }
12799         /*
12800          * User contexts:
12801          *      - default to 1 user context per real (non-HT) CPU core if
12802          *        num_user_contexts is negative
12803          */
12804         if (num_user_contexts < 0)
12805                 num_user_contexts =
12806                         cpumask_weight(&dd->affinity->real_cpu_mask);
12807
12808         total_contexts = num_kernel_contexts + num_user_contexts;
12809
12810         /*
12811          * Adjust the counts given a global max.
12812          */
12813         if (total_contexts > dd->chip_rcv_contexts) {
12814                 dd_dev_err(dd,
12815                            "Reducing # user receive contexts to: %d, from %d\n",
12816                            (int)(dd->chip_rcv_contexts - num_kernel_contexts),
12817                            (int)num_user_contexts);
12818                 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts;
12819                 /* recalculate */
12820                 total_contexts = num_kernel_contexts + num_user_contexts;
12821         }
12822
12823         /* each user context requires an entry in the RMT */
12824         qos_rmt_count = qos_rmt_entries(dd, NULL, NULL);
12825         if (qos_rmt_count + num_user_contexts > NUM_MAP_ENTRIES) {
12826                 user_rmt_reduced = NUM_MAP_ENTRIES - qos_rmt_count;
12827                 dd_dev_err(dd,
12828                            "RMT size is reducing the number of user receive contexts from %d to %d\n",
12829                            (int)num_user_contexts,
12830                            user_rmt_reduced);
12831                 /* recalculate */
12832                 num_user_contexts = user_rmt_reduced;
12833                 total_contexts = num_kernel_contexts + num_user_contexts;
12834         }
12835
12836         /* the first N are kernel contexts, the rest are user contexts */
12837         dd->num_rcv_contexts = total_contexts;
12838         dd->n_krcv_queues = num_kernel_contexts;
12839         dd->first_user_ctxt = num_kernel_contexts;
12840         dd->num_user_contexts = num_user_contexts;
12841         dd->freectxts = num_user_contexts;
12842         dd_dev_info(dd,
12843                     "rcv contexts: chip %d, used %d (kernel %d, user %d)\n",
12844                     (int)dd->chip_rcv_contexts,
12845                     (int)dd->num_rcv_contexts,
12846                     (int)dd->n_krcv_queues,
12847                     (int)dd->num_rcv_contexts - dd->n_krcv_queues);
12848
12849         /*
12850          * Receive array allocation:
12851          *   All RcvArray entries are divided into groups of 8. This
12852          *   is required by the hardware and will speed up writes to
12853          *   consecutive entries by using write-combining of the entire
12854          *   cacheline.
12855          *
12856          *   The number of groups are evenly divided among all contexts.
12857          *   any left over groups will be given to the first N user
12858          *   contexts.
12859          */
12860         dd->rcv_entries.group_size = RCV_INCREMENT;
12861         ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size;
12862         dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
12863         dd->rcv_entries.nctxt_extra = ngroups -
12864                 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
12865         dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
12866                     dd->rcv_entries.ngroups,
12867                     dd->rcv_entries.nctxt_extra);
12868         if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
12869             MAX_EAGER_ENTRIES * 2) {
12870                 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
12871                         dd->rcv_entries.group_size;
12872                 dd_dev_info(dd,
12873                             "RcvArray group count too high, change to %u\n",
12874                             dd->rcv_entries.ngroups);
12875                 dd->rcv_entries.nctxt_extra = 0;
12876         }
12877         /*
12878          * PIO send contexts
12879          */
12880         ret = init_sc_pools_and_sizes(dd);
12881         if (ret >= 0) { /* success */
12882                 dd->num_send_contexts = ret;
12883                 dd_dev_info(
12884                         dd,
12885                         "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
12886                         dd->chip_send_contexts,
12887                         dd->num_send_contexts,
12888                         dd->sc_sizes[SC_KERNEL].count,
12889                         dd->sc_sizes[SC_ACK].count,
12890                         dd->sc_sizes[SC_USER].count,
12891                         dd->sc_sizes[SC_VL15].count);
12892                 ret = 0;        /* success */
12893         }
12894
12895         return ret;
12896 }
12897
12898 /*
12899  * Set the device/port partition key table. The MAD code
12900  * will ensure that, at least, the partial management
12901  * partition key is present in the table.
12902  */
12903 static void set_partition_keys(struct hfi1_pportdata *ppd)
12904 {
12905         struct hfi1_devdata *dd = ppd->dd;
12906         u64 reg = 0;
12907         int i;
12908
12909         dd_dev_info(dd, "Setting partition keys\n");
12910         for (i = 0; i < hfi1_get_npkeys(dd); i++) {
12911                 reg |= (ppd->pkeys[i] &
12912                         RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
12913                         ((i % 4) *
12914                          RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
12915                 /* Each register holds 4 PKey values. */
12916                 if ((i % 4) == 3) {
12917                         write_csr(dd, RCV_PARTITION_KEY +
12918                                   ((i - 3) * 2), reg);
12919                         reg = 0;
12920                 }
12921         }
12922
12923         /* Always enable HW pkeys check when pkeys table is set */
12924         add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
12925 }
12926
12927 /*
12928  * These CSRs and memories are uninitialized on reset and must be
12929  * written before reading to set the ECC/parity bits.
12930  *
12931  * NOTE: All user context CSRs that are not mmaped write-only
12932  * (e.g. the TID flows) must be initialized even if the driver never
12933  * reads them.
12934  */
12935 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
12936 {
12937         int i, j;
12938
12939         /* CceIntMap */
12940         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
12941                 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
12942
12943         /* SendCtxtCreditReturnAddr */
12944         for (i = 0; i < dd->chip_send_contexts; i++)
12945                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
12946
12947         /* PIO Send buffers */
12948         /* SDMA Send buffers */
12949         /*
12950          * These are not normally read, and (presently) have no method
12951          * to be read, so are not pre-initialized
12952          */
12953
12954         /* RcvHdrAddr */
12955         /* RcvHdrTailAddr */
12956         /* RcvTidFlowTable */
12957         for (i = 0; i < dd->chip_rcv_contexts; i++) {
12958                 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
12959                 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
12960                 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
12961                         write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
12962         }
12963
12964         /* RcvArray */
12965         for (i = 0; i < dd->chip_rcv_array_count; i++)
12966                 write_csr(dd, RCV_ARRAY + (8 * i),
12967                           RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
12968
12969         /* RcvQPMapTable */
12970         for (i = 0; i < 32; i++)
12971                 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
12972 }
12973
12974 /*
12975  * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
12976  */
12977 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
12978                              u64 ctrl_bits)
12979 {
12980         unsigned long timeout;
12981         u64 reg;
12982
12983         /* is the condition present? */
12984         reg = read_csr(dd, CCE_STATUS);
12985         if ((reg & status_bits) == 0)
12986                 return;
12987
12988         /* clear the condition */
12989         write_csr(dd, CCE_CTRL, ctrl_bits);
12990
12991         /* wait for the condition to clear */
12992         timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
12993         while (1) {
12994                 reg = read_csr(dd, CCE_STATUS);
12995                 if ((reg & status_bits) == 0)
12996                         return;
12997                 if (time_after(jiffies, timeout)) {
12998                         dd_dev_err(dd,
12999                                    "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13000                                    status_bits, reg & status_bits);
13001                         return;
13002                 }
13003                 udelay(1);
13004         }
13005 }
13006
13007 /* set CCE CSRs to chip reset defaults */
13008 static void reset_cce_csrs(struct hfi1_devdata *dd)
13009 {
13010         int i;
13011
13012         /* CCE_REVISION read-only */
13013         /* CCE_REVISION2 read-only */
13014         /* CCE_CTRL - bits clear automatically */
13015         /* CCE_STATUS read-only, use CceCtrl to clear */
13016         clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13017         clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13018         clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13019         for (i = 0; i < CCE_NUM_SCRATCH; i++)
13020                 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13021         /* CCE_ERR_STATUS read-only */
13022         write_csr(dd, CCE_ERR_MASK, 0);
13023         write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13024         /* CCE_ERR_FORCE leave alone */
13025         for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13026                 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13027         write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13028         /* CCE_PCIE_CTRL leave alone */
13029         for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13030                 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13031                 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
13032                           CCE_MSIX_TABLE_UPPER_RESETCSR);
13033         }
13034         for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13035                 /* CCE_MSIX_PBA read-only */
13036                 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13037                 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13038         }
13039         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13040                 write_csr(dd, CCE_INT_MAP, 0);
13041         for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13042                 /* CCE_INT_STATUS read-only */
13043                 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13044                 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13045                 /* CCE_INT_FORCE leave alone */
13046                 /* CCE_INT_BLOCKED read-only */
13047         }
13048         for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13049                 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13050 }
13051
13052 /* set MISC CSRs to chip reset defaults */
13053 static void reset_misc_csrs(struct hfi1_devdata *dd)
13054 {
13055         int i;
13056
13057         for (i = 0; i < 32; i++) {
13058                 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13059                 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13060                 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13061         }
13062         /*
13063          * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13064          * only be written 128-byte chunks
13065          */
13066         /* init RSA engine to clear lingering errors */
13067         write_csr(dd, MISC_CFG_RSA_CMD, 1);
13068         write_csr(dd, MISC_CFG_RSA_MU, 0);
13069         write_csr(dd, MISC_CFG_FW_CTRL, 0);
13070         /* MISC_STS_8051_DIGEST read-only */
13071         /* MISC_STS_SBM_DIGEST read-only */
13072         /* MISC_STS_PCIE_DIGEST read-only */
13073         /* MISC_STS_FAB_DIGEST read-only */
13074         /* MISC_ERR_STATUS read-only */
13075         write_csr(dd, MISC_ERR_MASK, 0);
13076         write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13077         /* MISC_ERR_FORCE leave alone */
13078 }
13079
13080 /* set TXE CSRs to chip reset defaults */
13081 static void reset_txe_csrs(struct hfi1_devdata *dd)
13082 {
13083         int i;
13084
13085         /*
13086          * TXE Kernel CSRs
13087          */
13088         write_csr(dd, SEND_CTRL, 0);
13089         __cm_reset(dd, 0);      /* reset CM internal state */
13090         /* SEND_CONTEXTS read-only */
13091         /* SEND_DMA_ENGINES read-only */
13092         /* SEND_PIO_MEM_SIZE read-only */
13093         /* SEND_DMA_MEM_SIZE read-only */
13094         write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13095         pio_reset_all(dd);      /* SEND_PIO_INIT_CTXT */
13096         /* SEND_PIO_ERR_STATUS read-only */
13097         write_csr(dd, SEND_PIO_ERR_MASK, 0);
13098         write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13099         /* SEND_PIO_ERR_FORCE leave alone */
13100         /* SEND_DMA_ERR_STATUS read-only */
13101         write_csr(dd, SEND_DMA_ERR_MASK, 0);
13102         write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13103         /* SEND_DMA_ERR_FORCE leave alone */
13104         /* SEND_EGRESS_ERR_STATUS read-only */
13105         write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13106         write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13107         /* SEND_EGRESS_ERR_FORCE leave alone */
13108         write_csr(dd, SEND_BTH_QP, 0);
13109         write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13110         write_csr(dd, SEND_SC2VLT0, 0);
13111         write_csr(dd, SEND_SC2VLT1, 0);
13112         write_csr(dd, SEND_SC2VLT2, 0);
13113         write_csr(dd, SEND_SC2VLT3, 0);
13114         write_csr(dd, SEND_LEN_CHECK0, 0);
13115         write_csr(dd, SEND_LEN_CHECK1, 0);
13116         /* SEND_ERR_STATUS read-only */
13117         write_csr(dd, SEND_ERR_MASK, 0);
13118         write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13119         /* SEND_ERR_FORCE read-only */
13120         for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
13121                 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
13122         for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
13123                 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13124         for (i = 0; i < dd->chip_send_contexts / NUM_CONTEXTS_PER_SET; i++)
13125                 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
13126         for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
13127                 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
13128         for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
13129                 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
13130         write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
13131         write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
13132         /* SEND_CM_CREDIT_USED_STATUS read-only */
13133         write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13134         write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13135         write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13136         write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13137         write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13138         for (i = 0; i < TXE_NUM_DATA_VL; i++)
13139                 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
13140         write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13141         /* SEND_CM_CREDIT_USED_VL read-only */
13142         /* SEND_CM_CREDIT_USED_VL15 read-only */
13143         /* SEND_EGRESS_CTXT_STATUS read-only */
13144         /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13145         write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13146         /* SEND_EGRESS_ERR_INFO read-only */
13147         /* SEND_EGRESS_ERR_SOURCE read-only */
13148
13149         /*
13150          * TXE Per-Context CSRs
13151          */
13152         for (i = 0; i < dd->chip_send_contexts; i++) {
13153                 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13154                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13155                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13156                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13157                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13158                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13159                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13160                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13161                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13162                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13163                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13164                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13165         }
13166
13167         /*
13168          * TXE Per-SDMA CSRs
13169          */
13170         for (i = 0; i < dd->chip_sdma_engines; i++) {
13171                 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13172                 /* SEND_DMA_STATUS read-only */
13173                 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13174                 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13175                 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13176                 /* SEND_DMA_HEAD read-only */
13177                 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13178                 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13179                 /* SEND_DMA_IDLE_CNT read-only */
13180                 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13181                 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13182                 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13183                 /* SEND_DMA_ENG_ERR_STATUS read-only */
13184                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13185                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13186                 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13187                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13188                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13189                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13190                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13191                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13192                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13193                 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13194         }
13195 }
13196
13197 /*
13198  * Expect on entry:
13199  * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13200  */
13201 static void init_rbufs(struct hfi1_devdata *dd)
13202 {
13203         u64 reg;
13204         int count;
13205
13206         /*
13207          * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13208          * clear.
13209          */
13210         count = 0;
13211         while (1) {
13212                 reg = read_csr(dd, RCV_STATUS);
13213                 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13214                             | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13215                         break;
13216                 /*
13217                  * Give up after 1ms - maximum wait time.
13218                  *
13219                  * RBuf size is 148KiB.  Slowest possible is PCIe Gen1 x1 at
13220                  * 250MB/s bandwidth.  Lower rate to 66% for overhead to get:
13221                  *      148 KB / (66% * 250MB/s) = 920us
13222                  */
13223                 if (count++ > 500) {
13224                         dd_dev_err(dd,
13225                                    "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13226                                    __func__, reg);
13227                         break;
13228                 }
13229                 udelay(2); /* do not busy-wait the CSR */
13230         }
13231
13232         /* start the init - expect RcvCtrl to be 0 */
13233         write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13234
13235         /*
13236          * Read to force the write of Rcvtrl.RxRbufInit.  There is a brief
13237          * period after the write before RcvStatus.RxRbufInitDone is valid.
13238          * The delay in the first run through the loop below is sufficient and
13239          * required before the first read of RcvStatus.RxRbufInintDone.
13240          */
13241         read_csr(dd, RCV_CTRL);
13242
13243         /* wait for the init to finish */
13244         count = 0;
13245         while (1) {
13246                 /* delay is required first time through - see above */
13247                 udelay(2); /* do not busy-wait the CSR */
13248                 reg = read_csr(dd, RCV_STATUS);
13249                 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13250                         break;
13251
13252                 /* give up after 100us - slowest possible at 33MHz is 73us */
13253                 if (count++ > 50) {
13254                         dd_dev_err(dd,
13255                                    "%s: RcvStatus.RxRbufInit not set, continuing\n",
13256                                    __func__);
13257                         break;
13258                 }
13259         }
13260 }
13261
13262 /* set RXE CSRs to chip reset defaults */
13263 static void reset_rxe_csrs(struct hfi1_devdata *dd)
13264 {
13265         int i, j;
13266
13267         /*
13268          * RXE Kernel CSRs
13269          */
13270         write_csr(dd, RCV_CTRL, 0);
13271         init_rbufs(dd);
13272         /* RCV_STATUS read-only */
13273         /* RCV_CONTEXTS read-only */
13274         /* RCV_ARRAY_CNT read-only */
13275         /* RCV_BUF_SIZE read-only */
13276         write_csr(dd, RCV_BTH_QP, 0);
13277         write_csr(dd, RCV_MULTICAST, 0);
13278         write_csr(dd, RCV_BYPASS, 0);
13279         write_csr(dd, RCV_VL15, 0);
13280         /* this is a clear-down */
13281         write_csr(dd, RCV_ERR_INFO,
13282                   RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
13283         /* RCV_ERR_STATUS read-only */
13284         write_csr(dd, RCV_ERR_MASK, 0);
13285         write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13286         /* RCV_ERR_FORCE leave alone */
13287         for (i = 0; i < 32; i++)
13288                 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13289         for (i = 0; i < 4; i++)
13290                 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13291         for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13292                 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13293         for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13294                 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13295         for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) {
13296                 write_csr(dd, RCV_RSM_CFG + (8 * i), 0);
13297                 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0);
13298                 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0);
13299         }
13300         for (i = 0; i < 32; i++)
13301                 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13302
13303         /*
13304          * RXE Kernel and User Per-Context CSRs
13305          */
13306         for (i = 0; i < dd->chip_rcv_contexts; i++) {
13307                 /* kernel */
13308                 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13309                 /* RCV_CTXT_STATUS read-only */
13310                 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13311                 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13312                 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13313                 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13314                 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13315                 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13316                 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13317                 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13318                 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13319                 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13320
13321                 /* user */
13322                 /* RCV_HDR_TAIL read-only */
13323                 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13324                 /* RCV_EGR_INDEX_TAIL read-only */
13325                 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13326                 /* RCV_EGR_OFFSET_TAIL read-only */
13327                 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
13328                         write_uctxt_csr(dd, i,
13329                                         RCV_TID_FLOW_TABLE + (8 * j), 0);
13330                 }
13331         }
13332 }
13333
13334 /*
13335  * Set sc2vl tables.
13336  *
13337  * They power on to zeros, so to avoid send context errors
13338  * they need to be set:
13339  *
13340  * SC 0-7 -> VL 0-7 (respectively)
13341  * SC 15  -> VL 15
13342  * otherwise
13343  *        -> VL 0
13344  */
13345 static void init_sc2vl_tables(struct hfi1_devdata *dd)
13346 {
13347         int i;
13348         /* init per architecture spec, constrained by hardware capability */
13349
13350         /* HFI maps sent packets */
13351         write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13352                 0,
13353                 0, 0, 1, 1,
13354                 2, 2, 3, 3,
13355                 4, 4, 5, 5,
13356                 6, 6, 7, 7));
13357         write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13358                 1,
13359                 8, 0, 9, 0,
13360                 10, 0, 11, 0,
13361                 12, 0, 13, 0,
13362                 14, 0, 15, 15));
13363         write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13364                 2,
13365                 16, 0, 17, 0,
13366                 18, 0, 19, 0,
13367                 20, 0, 21, 0,
13368                 22, 0, 23, 0));
13369         write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13370                 3,
13371                 24, 0, 25, 0,
13372                 26, 0, 27, 0,
13373                 28, 0, 29, 0,
13374                 30, 0, 31, 0));
13375
13376         /* DC maps received packets */
13377         write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13378                 15_0,
13379                 0, 0, 1, 1,  2, 2,  3, 3,  4, 4,  5, 5,  6, 6,  7,  7,
13380                 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13381         write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13382                 31_16,
13383                 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13384                 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13385
13386         /* initialize the cached sc2vl values consistently with h/w */
13387         for (i = 0; i < 32; i++) {
13388                 if (i < 8 || i == 15)
13389                         *((u8 *)(dd->sc2vl) + i) = (u8)i;
13390                 else
13391                         *((u8 *)(dd->sc2vl) + i) = 0;
13392         }
13393 }
13394
13395 /*
13396  * Read chip sizes and then reset parts to sane, disabled, values.  We cannot
13397  * depend on the chip going through a power-on reset - a driver may be loaded
13398  * and unloaded many times.
13399  *
13400  * Do not write any CSR values to the chip in this routine - there may be
13401  * a reset following the (possible) FLR in this routine.
13402  *
13403  */
13404 static void init_chip(struct hfi1_devdata *dd)
13405 {
13406         int i;
13407
13408         /*
13409          * Put the HFI CSRs in a known state.
13410          * Combine this with a DC reset.
13411          *
13412          * Stop the device from doing anything while we do a
13413          * reset.  We know there are no other active users of
13414          * the device since we are now in charge.  Turn off
13415          * off all outbound and inbound traffic and make sure
13416          * the device does not generate any interrupts.
13417          */
13418
13419         /* disable send contexts and SDMA engines */
13420         write_csr(dd, SEND_CTRL, 0);
13421         for (i = 0; i < dd->chip_send_contexts; i++)
13422                 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13423         for (i = 0; i < dd->chip_sdma_engines; i++)
13424                 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13425         /* disable port (turn off RXE inbound traffic) and contexts */
13426         write_csr(dd, RCV_CTRL, 0);
13427         for (i = 0; i < dd->chip_rcv_contexts; i++)
13428                 write_csr(dd, RCV_CTXT_CTRL, 0);
13429         /* mask all interrupt sources */
13430         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13431                 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
13432
13433         /*
13434          * DC Reset: do a full DC reset before the register clear.
13435          * A recommended length of time to hold is one CSR read,
13436          * so reread the CceDcCtrl.  Then, hold the DC in reset
13437          * across the clear.
13438          */
13439         write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
13440         (void)read_csr(dd, CCE_DC_CTRL);
13441
13442         if (use_flr) {
13443                 /*
13444                  * A FLR will reset the SPC core and part of the PCIe.
13445                  * The parts that need to be restored have already been
13446                  * saved.
13447                  */
13448                 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13449
13450                 /* do the FLR, the DC reset will remain */
13451                 hfi1_pcie_flr(dd);
13452
13453                 /* restore command and BARs */
13454                 restore_pci_variables(dd);
13455
13456                 if (is_ax(dd)) {
13457                         dd_dev_info(dd, "Resetting CSRs with FLR\n");
13458                         hfi1_pcie_flr(dd);
13459                         restore_pci_variables(dd);
13460                 }
13461         } else {
13462                 dd_dev_info(dd, "Resetting CSRs with writes\n");
13463                 reset_cce_csrs(dd);
13464                 reset_txe_csrs(dd);
13465                 reset_rxe_csrs(dd);
13466                 reset_misc_csrs(dd);
13467         }
13468         /* clear the DC reset */
13469         write_csr(dd, CCE_DC_CTRL, 0);
13470
13471         /* Set the LED off */
13472         setextled(dd, 0);
13473
13474         /*
13475          * Clear the QSFP reset.
13476          * An FLR enforces a 0 on all out pins. The driver does not touch
13477          * ASIC_QSFPn_OUT otherwise.  This leaves RESET_N low and
13478          * anything plugged constantly in reset, if it pays attention
13479          * to RESET_N.
13480          * Prime examples of this are optical cables. Set all pins high.
13481          * I2CCLK and I2CDAT will change per direction, and INT_N and
13482          * MODPRS_N are input only and their value is ignored.
13483          */
13484         write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
13485         write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
13486         init_chip_resources(dd);
13487 }
13488
13489 static void init_early_variables(struct hfi1_devdata *dd)
13490 {
13491         int i;
13492
13493         /* assign link credit variables */
13494         dd->vau = CM_VAU;
13495         dd->link_credits = CM_GLOBAL_CREDITS;
13496         if (is_ax(dd))
13497                 dd->link_credits--;
13498         dd->vcu = cu_to_vcu(hfi1_cu);
13499         /* enough room for 8 MAD packets plus header - 17K */
13500         dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
13501         if (dd->vl15_init > dd->link_credits)
13502                 dd->vl15_init = dd->link_credits;
13503
13504         write_uninitialized_csrs_and_memories(dd);
13505
13506         if (HFI1_CAP_IS_KSET(PKEY_CHECK))
13507                 for (i = 0; i < dd->num_pports; i++) {
13508                         struct hfi1_pportdata *ppd = &dd->pport[i];
13509
13510                         set_partition_keys(ppd);
13511                 }
13512         init_sc2vl_tables(dd);
13513 }
13514
13515 static void init_kdeth_qp(struct hfi1_devdata *dd)
13516 {
13517         /* user changed the KDETH_QP */
13518         if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
13519                 /* out of range or illegal value */
13520                 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
13521                 kdeth_qp = 0;
13522         }
13523         if (kdeth_qp == 0)      /* not set, or failed range check */
13524                 kdeth_qp = DEFAULT_KDETH_QP;
13525
13526         write_csr(dd, SEND_BTH_QP,
13527                   (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
13528                   SEND_BTH_QP_KDETH_QP_SHIFT);
13529
13530         write_csr(dd, RCV_BTH_QP,
13531                   (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
13532                   RCV_BTH_QP_KDETH_QP_SHIFT);
13533 }
13534
13535 /**
13536  * init_qpmap_table
13537  * @dd - device data
13538  * @first_ctxt - first context
13539  * @last_ctxt - first context
13540  *
13541  * This return sets the qpn mapping table that
13542  * is indexed by qpn[8:1].
13543  *
13544  * The routine will round robin the 256 settings
13545  * from first_ctxt to last_ctxt.
13546  *
13547  * The first/last looks ahead to having specialized
13548  * receive contexts for mgmt and bypass.  Normal
13549  * verbs traffic will assumed to be on a range
13550  * of receive contexts.
13551  */
13552 static void init_qpmap_table(struct hfi1_devdata *dd,
13553                              u32 first_ctxt,
13554                              u32 last_ctxt)
13555 {
13556         u64 reg = 0;
13557         u64 regno = RCV_QP_MAP_TABLE;
13558         int i;
13559         u64 ctxt = first_ctxt;
13560
13561         for (i = 0; i < 256; i++) {
13562                 reg |= ctxt << (8 * (i % 8));
13563                 ctxt++;
13564                 if (ctxt > last_ctxt)
13565                         ctxt = first_ctxt;
13566                 if (i % 8 == 7) {
13567                         write_csr(dd, regno, reg);
13568                         reg = 0;
13569                         regno += 8;
13570                 }
13571         }
13572
13573         add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
13574                         | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
13575 }
13576
13577 struct rsm_map_table {
13578         u64 map[NUM_MAP_REGS];
13579         unsigned int used;
13580 };
13581
13582 struct rsm_rule_data {
13583         u8 offset;
13584         u8 pkt_type;
13585         u32 field1_off;
13586         u32 field2_off;
13587         u32 index1_off;
13588         u32 index1_width;
13589         u32 index2_off;
13590         u32 index2_width;
13591         u32 mask1;
13592         u32 value1;
13593         u32 mask2;
13594         u32 value2;
13595 };
13596
13597 /*
13598  * Return an initialized RMT map table for users to fill in.  OK if it
13599  * returns NULL, indicating no table.
13600  */
13601 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
13602 {
13603         struct rsm_map_table *rmt;
13604         u8 rxcontext = is_ax(dd) ? 0 : 0xff;  /* 0 is default if a0 ver. */
13605
13606         rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
13607         if (rmt) {
13608                 memset(rmt->map, rxcontext, sizeof(rmt->map));
13609                 rmt->used = 0;
13610         }
13611
13612         return rmt;
13613 }
13614
13615 /*
13616  * Write the final RMT map table to the chip and free the table.  OK if
13617  * table is NULL.
13618  */
13619 static void complete_rsm_map_table(struct hfi1_devdata *dd,
13620                                    struct rsm_map_table *rmt)
13621 {
13622         int i;
13623
13624         if (rmt) {
13625                 /* write table to chip */
13626                 for (i = 0; i < NUM_MAP_REGS; i++)
13627                         write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
13628
13629                 /* enable RSM */
13630                 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
13631         }
13632 }
13633
13634 /*
13635  * Add a receive side mapping rule.
13636  */
13637 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
13638                          struct rsm_rule_data *rrd)
13639 {
13640         write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
13641                   (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
13642                   1ull << rule_index | /* enable bit */
13643                   (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
13644         write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
13645                   (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
13646                   (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
13647                   (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
13648                   (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
13649                   (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
13650                   (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
13651         write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
13652                   (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
13653                   (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
13654                   (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
13655                   (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
13656 }
13657
13658 /* return the number of RSM map table entries that will be used for QOS */
13659 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
13660                            unsigned int *np)
13661 {
13662         int i;
13663         unsigned int m, n;
13664         u8 max_by_vl = 0;
13665
13666         /* is QOS active at all? */
13667         if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
13668             num_vls == 1 ||
13669             krcvqsset <= 1)
13670                 goto no_qos;
13671
13672         /* determine bits for qpn */
13673         for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
13674                 if (krcvqs[i] > max_by_vl)
13675                         max_by_vl = krcvqs[i];
13676         if (max_by_vl > 32)
13677                 goto no_qos;
13678         m = ilog2(__roundup_pow_of_two(max_by_vl));
13679
13680         /* determine bits for vl */
13681         n = ilog2(__roundup_pow_of_two(num_vls));
13682
13683         /* reject if too much is used */
13684         if ((m + n) > 7)
13685                 goto no_qos;
13686
13687         if (mp)
13688                 *mp = m;
13689         if (np)
13690                 *np = n;
13691
13692         return 1 << (m + n);
13693
13694 no_qos:
13695         if (mp)
13696                 *mp = 0;
13697         if (np)
13698                 *np = 0;
13699         return 0;
13700 }
13701
13702 /**
13703  * init_qos - init RX qos
13704  * @dd - device data
13705  * @rmt - RSM map table
13706  *
13707  * This routine initializes Rule 0 and the RSM map table to implement
13708  * quality of service (qos).
13709  *
13710  * If all of the limit tests succeed, qos is applied based on the array
13711  * interpretation of krcvqs where entry 0 is VL0.
13712  *
13713  * The number of vl bits (n) and the number of qpn bits (m) are computed to
13714  * feed both the RSM map table and the single rule.
13715  */
13716 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
13717 {
13718         struct rsm_rule_data rrd;
13719         unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
13720         unsigned int rmt_entries;
13721         u64 reg;
13722
13723         if (!rmt)
13724                 goto bail;
13725         rmt_entries = qos_rmt_entries(dd, &m, &n);
13726         if (rmt_entries == 0)
13727                 goto bail;
13728         qpns_per_vl = 1 << m;
13729
13730         /* enough room in the map table? */
13731         rmt_entries = 1 << (m + n);
13732         if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
13733                 goto bail;
13734
13735         /* add qos entries to the the RSM map table */
13736         for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
13737                 unsigned tctxt;
13738
13739                 for (qpn = 0, tctxt = ctxt;
13740                      krcvqs[i] && qpn < qpns_per_vl; qpn++) {
13741                         unsigned idx, regoff, regidx;
13742
13743                         /* generate the index the hardware will produce */
13744                         idx = rmt->used + ((qpn << n) ^ i);
13745                         regoff = (idx % 8) * 8;
13746                         regidx = idx / 8;
13747                         /* replace default with context number */
13748                         reg = rmt->map[regidx];
13749                         reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
13750                                 << regoff);
13751                         reg |= (u64)(tctxt++) << regoff;
13752                         rmt->map[regidx] = reg;
13753                         if (tctxt == ctxt + krcvqs[i])
13754                                 tctxt = ctxt;
13755                 }
13756                 ctxt += krcvqs[i];
13757         }
13758
13759         rrd.offset = rmt->used;
13760         rrd.pkt_type = 2;
13761         rrd.field1_off = LRH_BTH_MATCH_OFFSET;
13762         rrd.field2_off = LRH_SC_MATCH_OFFSET;
13763         rrd.index1_off = LRH_SC_SELECT_OFFSET;
13764         rrd.index1_width = n;
13765         rrd.index2_off = QPN_SELECT_OFFSET;
13766         rrd.index2_width = m + n;
13767         rrd.mask1 = LRH_BTH_MASK;
13768         rrd.value1 = LRH_BTH_VALUE;
13769         rrd.mask2 = LRH_SC_MASK;
13770         rrd.value2 = LRH_SC_VALUE;
13771
13772         /* add rule 0 */
13773         add_rsm_rule(dd, 0, &rrd);
13774
13775         /* mark RSM map entries as used */
13776         rmt->used += rmt_entries;
13777         /* map everything else to the mcast/err/vl15 context */
13778         init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
13779         dd->qos_shift = n + 1;
13780         return;
13781 bail:
13782         dd->qos_shift = 1;
13783         init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
13784 }
13785
13786 static void init_user_fecn_handling(struct hfi1_devdata *dd,
13787                                     struct rsm_map_table *rmt)
13788 {
13789         struct rsm_rule_data rrd;
13790         u64 reg;
13791         int i, idx, regoff, regidx;
13792         u8 offset;
13793
13794         /* there needs to be enough room in the map table */
13795         if (rmt->used + dd->num_user_contexts >= NUM_MAP_ENTRIES) {
13796                 dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n");
13797                 return;
13798         }
13799
13800         /*
13801          * RSM will extract the destination context as an index into the
13802          * map table.  The destination contexts are a sequential block
13803          * in the range first_user_ctxt...num_rcv_contexts-1 (inclusive).
13804          * Map entries are accessed as offset + extracted value.  Adjust
13805          * the added offset so this sequence can be placed anywhere in
13806          * the table - as long as the entries themselves do not wrap.
13807          * There are only enough bits in offset for the table size, so
13808          * start with that to allow for a "negative" offset.
13809          */
13810         offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used -
13811                                                 (int)dd->first_user_ctxt);
13812
13813         for (i = dd->first_user_ctxt, idx = rmt->used;
13814                                 i < dd->num_rcv_contexts; i++, idx++) {
13815                 /* replace with identity mapping */
13816                 regoff = (idx % 8) * 8;
13817                 regidx = idx / 8;
13818                 reg = rmt->map[regidx];
13819                 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
13820                 reg |= (u64)i << regoff;
13821                 rmt->map[regidx] = reg;
13822         }
13823
13824         /*
13825          * For RSM intercept of Expected FECN packets:
13826          * o packet type 0 - expected
13827          * o match on F (bit 95), using select/match 1, and
13828          * o match on SH (bit 133), using select/match 2.
13829          *
13830          * Use index 1 to extract the 8-bit receive context from DestQP
13831          * (start at bit 64).  Use that as the RSM map table index.
13832          */
13833         rrd.offset = offset;
13834         rrd.pkt_type = 0;
13835         rrd.field1_off = 95;
13836         rrd.field2_off = 133;
13837         rrd.index1_off = 64;
13838         rrd.index1_width = 8;
13839         rrd.index2_off = 0;
13840         rrd.index2_width = 0;
13841         rrd.mask1 = 1;
13842         rrd.value1 = 1;
13843         rrd.mask2 = 1;
13844         rrd.value2 = 1;
13845
13846         /* add rule 1 */
13847         add_rsm_rule(dd, 1, &rrd);
13848
13849         rmt->used += dd->num_user_contexts;
13850 }
13851
13852 static void init_rxe(struct hfi1_devdata *dd)
13853 {
13854         struct rsm_map_table *rmt;
13855
13856         /* enable all receive errors */
13857         write_csr(dd, RCV_ERR_MASK, ~0ull);
13858
13859         rmt = alloc_rsm_map_table(dd);
13860         /* set up QOS, including the QPN map table */
13861         init_qos(dd, rmt);
13862         init_user_fecn_handling(dd, rmt);
13863         complete_rsm_map_table(dd, rmt);
13864         kfree(rmt);
13865
13866         /*
13867          * make sure RcvCtrl.RcvWcb <= PCIe Device Control
13868          * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
13869          * space, PciCfgCap2.MaxPayloadSize in HFI).  There is only one
13870          * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
13871          * Max_PayLoad_Size set to its minimum of 128.
13872          *
13873          * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
13874          * (64 bytes).  Max_Payload_Size is possibly modified upward in
13875          * tune_pcie_caps() which is called after this routine.
13876          */
13877 }
13878
13879 static void init_other(struct hfi1_devdata *dd)
13880 {
13881         /* enable all CCE errors */
13882         write_csr(dd, CCE_ERR_MASK, ~0ull);
13883         /* enable *some* Misc errors */
13884         write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
13885         /* enable all DC errors, except LCB */
13886         write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
13887         write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
13888 }
13889
13890 /*
13891  * Fill out the given AU table using the given CU.  A CU is defined in terms
13892  * AUs.  The table is a an encoding: given the index, how many AUs does that
13893  * represent?
13894  *
13895  * NOTE: Assumes that the register layout is the same for the
13896  * local and remote tables.
13897  */
13898 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
13899                                u32 csr0to3, u32 csr4to7)
13900 {
13901         write_csr(dd, csr0to3,
13902                   0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
13903                   1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
13904                   2ull * cu <<
13905                   SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
13906                   4ull * cu <<
13907                   SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
13908         write_csr(dd, csr4to7,
13909                   8ull * cu <<
13910                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
13911                   16ull * cu <<
13912                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
13913                   32ull * cu <<
13914                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
13915                   64ull * cu <<
13916                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
13917 }
13918
13919 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
13920 {
13921         assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
13922                            SEND_CM_LOCAL_AU_TABLE4_TO7);
13923 }
13924
13925 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
13926 {
13927         assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
13928                            SEND_CM_REMOTE_AU_TABLE4_TO7);
13929 }
13930
13931 static void init_txe(struct hfi1_devdata *dd)
13932 {
13933         int i;
13934
13935         /* enable all PIO, SDMA, general, and Egress errors */
13936         write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
13937         write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
13938         write_csr(dd, SEND_ERR_MASK, ~0ull);
13939         write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
13940
13941         /* enable all per-context and per-SDMA engine errors */
13942         for (i = 0; i < dd->chip_send_contexts; i++)
13943                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
13944         for (i = 0; i < dd->chip_sdma_engines; i++)
13945                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
13946
13947         /* set the local CU to AU mapping */
13948         assign_local_cm_au_table(dd, dd->vcu);
13949
13950         /*
13951          * Set reasonable default for Credit Return Timer
13952          * Don't set on Simulator - causes it to choke.
13953          */
13954         if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
13955                 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
13956 }
13957
13958 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
13959 {
13960         struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
13961         unsigned sctxt;
13962         int ret = 0;
13963         u64 reg;
13964
13965         if (!rcd || !rcd->sc) {
13966                 ret = -EINVAL;
13967                 goto done;
13968         }
13969         sctxt = rcd->sc->hw_context;
13970         reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
13971                 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
13972                  SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
13973         /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
13974         if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
13975                 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
13976         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
13977         /*
13978          * Enable send-side J_KEY integrity check, unless this is A0 h/w
13979          */
13980         if (!is_ax(dd)) {
13981                 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
13982                 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
13983                 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
13984         }
13985
13986         /* Enable J_KEY check on receive context. */
13987         reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
13988                 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
13989                  RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
13990         write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
13991 done:
13992         return ret;
13993 }
13994
13995 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
13996 {
13997         struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
13998         unsigned sctxt;
13999         int ret = 0;
14000         u64 reg;
14001
14002         if (!rcd || !rcd->sc) {
14003                 ret = -EINVAL;
14004                 goto done;
14005         }
14006         sctxt = rcd->sc->hw_context;
14007         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14008         /*
14009          * Disable send-side J_KEY integrity check, unless this is A0 h/w.
14010          * This check would not have been enabled for A0 h/w, see
14011          * set_ctxt_jkey().
14012          */
14013         if (!is_ax(dd)) {
14014                 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14015                 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14016                 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14017         }
14018         /* Turn off the J_KEY on the receive side */
14019         write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
14020 done:
14021         return ret;
14022 }
14023
14024 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
14025 {
14026         struct hfi1_ctxtdata *rcd;
14027         unsigned sctxt;
14028         int ret = 0;
14029         u64 reg;
14030
14031         if (ctxt < dd->num_rcv_contexts) {
14032                 rcd = dd->rcd[ctxt];
14033         } else {
14034                 ret = -EINVAL;
14035                 goto done;
14036         }
14037         if (!rcd || !rcd->sc) {
14038                 ret = -EINVAL;
14039                 goto done;
14040         }
14041         sctxt = rcd->sc->hw_context;
14042         reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14043                 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14044         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14045         reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14046         reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14047         reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
14048         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14049 done:
14050         return ret;
14051 }
14052
14053 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt)
14054 {
14055         struct hfi1_ctxtdata *rcd;
14056         unsigned sctxt;
14057         int ret = 0;
14058         u64 reg;
14059
14060         if (ctxt < dd->num_rcv_contexts) {
14061                 rcd = dd->rcd[ctxt];
14062         } else {
14063                 ret = -EINVAL;
14064                 goto done;
14065         }
14066         if (!rcd || !rcd->sc) {
14067                 ret = -EINVAL;
14068                 goto done;
14069         }
14070         sctxt = rcd->sc->hw_context;
14071         reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
14072         reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14073         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
14074         write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14075 done:
14076         return ret;
14077 }
14078
14079 /*
14080  * Start doing the clean up the the chip. Our clean up happens in multiple
14081  * stages and this is just the first.
14082  */
14083 void hfi1_start_cleanup(struct hfi1_devdata *dd)
14084 {
14085         aspm_exit(dd);
14086         free_cntrs(dd);
14087         free_rcverr(dd);
14088         clean_up_interrupts(dd);
14089         finish_chip_resources(dd);
14090 }
14091
14092 #define HFI_BASE_GUID(dev) \
14093         ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14094
14095 /*
14096  * Information can be shared between the two HFIs on the same ASIC
14097  * in the same OS.  This function finds the peer device and sets
14098  * up a shared structure.
14099  */
14100 static int init_asic_data(struct hfi1_devdata *dd)
14101 {
14102         unsigned long flags;
14103         struct hfi1_devdata *tmp, *peer = NULL;
14104         int ret = 0;
14105
14106         spin_lock_irqsave(&hfi1_devs_lock, flags);
14107         /* Find our peer device */
14108         list_for_each_entry(tmp, &hfi1_dev_list, list) {
14109                 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
14110                     dd->unit != tmp->unit) {
14111                         peer = tmp;
14112                         break;
14113                 }
14114         }
14115
14116         if (peer) {
14117                 dd->asic_data = peer->asic_data;
14118         } else {
14119                 dd->asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14120                 if (!dd->asic_data) {
14121                         ret = -ENOMEM;
14122                         goto done;
14123                 }
14124                 mutex_init(&dd->asic_data->asic_resource_mutex);
14125         }
14126         dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
14127
14128 done:
14129         spin_unlock_irqrestore(&hfi1_devs_lock, flags);
14130         return ret;
14131 }
14132
14133 /*
14134  * Set dd->boardname.  Use a generic name if a name is not returned from
14135  * EFI variable space.
14136  *
14137  * Return 0 on success, -ENOMEM if space could not be allocated.
14138  */
14139 static int obtain_boardname(struct hfi1_devdata *dd)
14140 {
14141         /* generic board description */
14142         const char generic[] =
14143                 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14144         unsigned long size;
14145         int ret;
14146
14147         ret = read_hfi1_efi_var(dd, "description", &size,
14148                                 (void **)&dd->boardname);
14149         if (ret) {
14150                 dd_dev_info(dd, "Board description not found\n");
14151                 /* use generic description */
14152                 dd->boardname = kstrdup(generic, GFP_KERNEL);
14153                 if (!dd->boardname)
14154                         return -ENOMEM;
14155         }
14156         return 0;
14157 }
14158
14159 /*
14160  * Check the interrupt registers to make sure that they are mapped correctly.
14161  * It is intended to help user identify any mismapping by VMM when the driver
14162  * is running in a VM. This function should only be called before interrupt
14163  * is set up properly.
14164  *
14165  * Return 0 on success, -EINVAL on failure.
14166  */
14167 static int check_int_registers(struct hfi1_devdata *dd)
14168 {
14169         u64 reg;
14170         u64 all_bits = ~(u64)0;
14171         u64 mask;
14172
14173         /* Clear CceIntMask[0] to avoid raising any interrupts */
14174         mask = read_csr(dd, CCE_INT_MASK);
14175         write_csr(dd, CCE_INT_MASK, 0ull);
14176         reg = read_csr(dd, CCE_INT_MASK);
14177         if (reg)
14178                 goto err_exit;
14179
14180         /* Clear all interrupt status bits */
14181         write_csr(dd, CCE_INT_CLEAR, all_bits);
14182         reg = read_csr(dd, CCE_INT_STATUS);
14183         if (reg)
14184                 goto err_exit;
14185
14186         /* Set all interrupt status bits */
14187         write_csr(dd, CCE_INT_FORCE, all_bits);
14188         reg = read_csr(dd, CCE_INT_STATUS);
14189         if (reg != all_bits)
14190                 goto err_exit;
14191
14192         /* Restore the interrupt mask */
14193         write_csr(dd, CCE_INT_CLEAR, all_bits);
14194         write_csr(dd, CCE_INT_MASK, mask);
14195
14196         return 0;
14197 err_exit:
14198         write_csr(dd, CCE_INT_MASK, mask);
14199         dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14200         return -EINVAL;
14201 }
14202
14203 /**
14204  * Allocate and initialize the device structure for the hfi.
14205  * @dev: the pci_dev for hfi1_ib device
14206  * @ent: pci_device_id struct for this dev
14207  *
14208  * Also allocates, initializes, and returns the devdata struct for this
14209  * device instance
14210  *
14211  * This is global, and is called directly at init to set up the
14212  * chip-specific function pointers for later use.
14213  */
14214 struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
14215                                   const struct pci_device_id *ent)
14216 {
14217         struct hfi1_devdata *dd;
14218         struct hfi1_pportdata *ppd;
14219         u64 reg;
14220         int i, ret;
14221         static const char * const inames[] = { /* implementation names */
14222                 "RTL silicon",
14223                 "RTL VCS simulation",
14224                 "RTL FPGA emulation",
14225                 "Functional simulator"
14226         };
14227         struct pci_dev *parent = pdev->bus->self;
14228
14229         dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
14230                                 sizeof(struct hfi1_pportdata));
14231         if (IS_ERR(dd))
14232                 goto bail;
14233         ppd = dd->pport;
14234         for (i = 0; i < dd->num_pports; i++, ppd++) {
14235                 int vl;
14236                 /* init common fields */
14237                 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
14238                 /* DC supports 4 link widths */
14239                 ppd->link_width_supported =
14240                         OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
14241                         OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
14242                 ppd->link_width_downgrade_supported =
14243                         ppd->link_width_supported;
14244                 /* start out enabling only 4X */
14245                 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
14246                 ppd->link_width_downgrade_enabled =
14247                                         ppd->link_width_downgrade_supported;
14248                 /* link width active is 0 when link is down */
14249                 /* link width downgrade active is 0 when link is down */
14250
14251                 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
14252                     num_vls > HFI1_MAX_VLS_SUPPORTED) {
14253                         hfi1_early_err(&pdev->dev,
14254                                        "Invalid num_vls %u, using %u VLs\n",
14255                                     num_vls, HFI1_MAX_VLS_SUPPORTED);
14256                         num_vls = HFI1_MAX_VLS_SUPPORTED;
14257                 }
14258                 ppd->vls_supported = num_vls;
14259                 ppd->vls_operational = ppd->vls_supported;
14260                 ppd->actual_vls_operational = ppd->vls_supported;
14261                 /* Set the default MTU. */
14262                 for (vl = 0; vl < num_vls; vl++)
14263                         dd->vld[vl].mtu = hfi1_max_mtu;
14264                 dd->vld[15].mtu = MAX_MAD_PACKET;
14265                 /*
14266                  * Set the initial values to reasonable default, will be set
14267                  * for real when link is up.
14268                  */
14269                 ppd->lstate = IB_PORT_DOWN;
14270                 ppd->overrun_threshold = 0x4;
14271                 ppd->phy_error_threshold = 0xf;
14272                 ppd->port_crc_mode_enabled = link_crc_mask;
14273                 /* initialize supported LTP CRC mode */
14274                 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
14275                 /* initialize enabled LTP CRC mode */
14276                 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
14277                 /* start in offline */
14278                 ppd->host_link_state = HLS_DN_OFFLINE;
14279                 init_vl_arb_caches(ppd);
14280                 ppd->last_pstate = 0xff; /* invalid value */
14281         }
14282
14283         dd->link_default = HLS_DN_POLL;
14284
14285         /*
14286          * Do remaining PCIe setup and save PCIe values in dd.
14287          * Any error printing is already done by the init code.
14288          * On return, we have the chip mapped.
14289          */
14290         ret = hfi1_pcie_ddinit(dd, pdev, ent);
14291         if (ret < 0)
14292                 goto bail_free;
14293
14294         /* verify that reads actually work, save revision for reset check */
14295         dd->revision = read_csr(dd, CCE_REVISION);
14296         if (dd->revision == ~(u64)0) {
14297                 dd_dev_err(dd, "cannot read chip CSRs\n");
14298                 ret = -EINVAL;
14299                 goto bail_cleanup;
14300         }
14301         dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
14302                         & CCE_REVISION_CHIP_REV_MAJOR_MASK;
14303         dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
14304                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
14305
14306         /*
14307          * Check interrupt registers mapping if the driver has no access to
14308          * the upstream component. In this case, it is likely that the driver
14309          * is running in a VM.
14310          */
14311         if (!parent) {
14312                 ret = check_int_registers(dd);
14313                 if (ret)
14314                         goto bail_cleanup;
14315         }
14316
14317         /*
14318          * obtain the hardware ID - NOT related to unit, which is a
14319          * software enumeration
14320          */
14321         reg = read_csr(dd, CCE_REVISION2);
14322         dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
14323                                         & CCE_REVISION2_HFI_ID_MASK;
14324         /* the variable size will remove unwanted bits */
14325         dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
14326         dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
14327         dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
14328                     dd->icode < ARRAY_SIZE(inames) ?
14329                     inames[dd->icode] : "unknown", (int)dd->irev);
14330
14331         /* speeds the hardware can support */
14332         dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
14333         /* speeds allowed to run at */
14334         dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
14335         /* give a reasonable active value, will be set on link up */
14336         dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
14337
14338         dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS);
14339         dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS);
14340         dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES);
14341         dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE);
14342         dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE);
14343         /* fix up link widths for emulation _p */
14344         ppd = dd->pport;
14345         if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
14346                 ppd->link_width_supported =
14347                         ppd->link_width_enabled =
14348                         ppd->link_width_downgrade_supported =
14349                         ppd->link_width_downgrade_enabled =
14350                                 OPA_LINK_WIDTH_1X;
14351         }
14352         /* insure num_vls isn't larger than number of sdma engines */
14353         if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) {
14354                 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
14355                            num_vls, dd->chip_sdma_engines);
14356                 num_vls = dd->chip_sdma_engines;
14357                 ppd->vls_supported = dd->chip_sdma_engines;
14358                 ppd->vls_operational = ppd->vls_supported;
14359         }
14360
14361         /*
14362          * Convert the ns parameter to the 64 * cclocks used in the CSR.
14363          * Limit the max if larger than the field holds.  If timeout is
14364          * non-zero, then the calculated field will be at least 1.
14365          *
14366          * Must be after icode is set up - the cclock rate depends
14367          * on knowing the hardware being used.
14368          */
14369         dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
14370         if (dd->rcv_intr_timeout_csr >
14371                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
14372                 dd->rcv_intr_timeout_csr =
14373                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
14374         else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
14375                 dd->rcv_intr_timeout_csr = 1;
14376
14377         /* needs to be done before we look for the peer device */
14378         read_guid(dd);
14379
14380         /* set up shared ASIC data with peer device */
14381         ret = init_asic_data(dd);
14382         if (ret)
14383                 goto bail_cleanup;
14384
14385         /* obtain chip sizes, reset chip CSRs */
14386         init_chip(dd);
14387
14388         /* read in the PCIe link speed information */
14389         ret = pcie_speeds(dd);
14390         if (ret)
14391                 goto bail_cleanup;
14392
14393         /* Needs to be called before hfi1_firmware_init */
14394         get_platform_config(dd);
14395
14396         /* read in firmware */
14397         ret = hfi1_firmware_init(dd);
14398         if (ret)
14399                 goto bail_cleanup;
14400
14401         /*
14402          * In general, the PCIe Gen3 transition must occur after the
14403          * chip has been idled (so it won't initiate any PCIe transactions
14404          * e.g. an interrupt) and before the driver changes any registers
14405          * (the transition will reset the registers).
14406          *
14407          * In particular, place this call after:
14408          * - init_chip()     - the chip will not initiate any PCIe transactions
14409          * - pcie_speeds()   - reads the current link speed
14410          * - hfi1_firmware_init() - the needed firmware is ready to be
14411          *                          downloaded
14412          */
14413         ret = do_pcie_gen3_transition(dd);
14414         if (ret)
14415                 goto bail_cleanup;
14416
14417         /* start setting dd values and adjusting CSRs */
14418         init_early_variables(dd);
14419
14420         parse_platform_config(dd);
14421
14422         ret = obtain_boardname(dd);
14423         if (ret)
14424                 goto bail_cleanup;
14425
14426         snprintf(dd->boardversion, BOARD_VERS_MAX,
14427                  "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
14428                  HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
14429                  (u32)dd->majrev,
14430                  (u32)dd->minrev,
14431                  (dd->revision >> CCE_REVISION_SW_SHIFT)
14432                     & CCE_REVISION_SW_MASK);
14433
14434         /*
14435          * The real cpu mask is part of the affinity struct but has to be
14436          * initialized earlier than the rest of the affinity struct because it
14437          * is needed to calculate the number of user contexts in
14438          * set_up_context_variables(). However, hfi1_dev_affinity_init(),
14439          * which initializes the rest of the affinity struct members,
14440          * depends on set_up_context_variables() for the number of kernel
14441          * contexts, so it cannot be called before set_up_context_variables().
14442          */
14443         ret = init_real_cpu_mask(dd);
14444         if (ret)
14445                 goto bail_cleanup;
14446
14447         ret = set_up_context_variables(dd);
14448         if (ret)
14449                 goto bail_cleanup;
14450
14451         /* set initial RXE CSRs */
14452         init_rxe(dd);
14453         /* set initial TXE CSRs */
14454         init_txe(dd);
14455         /* set initial non-RXE, non-TXE CSRs */
14456         init_other(dd);
14457         /* set up KDETH QP prefix in both RX and TX CSRs */
14458         init_kdeth_qp(dd);
14459
14460         hfi1_dev_affinity_init(dd);
14461
14462         /* send contexts must be set up before receive contexts */
14463         ret = init_send_contexts(dd);
14464         if (ret)
14465                 goto bail_cleanup;
14466
14467         ret = hfi1_create_ctxts(dd);
14468         if (ret)
14469                 goto bail_cleanup;
14470
14471         dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
14472         /*
14473          * rcd[0] is guaranteed to be valid by this point. Also, all
14474          * context are using the same value, as per the module parameter.
14475          */
14476         dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
14477
14478         ret = init_pervl_scs(dd);
14479         if (ret)
14480                 goto bail_cleanup;
14481
14482         /* sdma init */
14483         for (i = 0; i < dd->num_pports; ++i) {
14484                 ret = sdma_init(dd, i);
14485                 if (ret)
14486                         goto bail_cleanup;
14487         }
14488
14489         /* use contexts created by hfi1_create_ctxts */
14490         ret = set_up_interrupts(dd);
14491         if (ret)
14492                 goto bail_cleanup;
14493
14494         /* set up LCB access - must be after set_up_interrupts() */
14495         init_lcb_access(dd);
14496
14497         snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
14498                  dd->base_guid & 0xFFFFFF);
14499
14500         dd->oui1 = dd->base_guid >> 56 & 0xFF;
14501         dd->oui2 = dd->base_guid >> 48 & 0xFF;
14502         dd->oui3 = dd->base_guid >> 40 & 0xFF;
14503
14504         ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
14505         if (ret)
14506                 goto bail_clear_intr;
14507         check_fabric_firmware_versions(dd);
14508
14509         thermal_init(dd);
14510
14511         ret = init_cntrs(dd);
14512         if (ret)
14513                 goto bail_clear_intr;
14514
14515         ret = init_rcverr(dd);
14516         if (ret)
14517                 goto bail_free_cntrs;
14518
14519         ret = eprom_init(dd);
14520         if (ret)
14521                 goto bail_free_rcverr;
14522
14523         goto bail;
14524
14525 bail_free_rcverr:
14526         free_rcverr(dd);
14527 bail_free_cntrs:
14528         free_cntrs(dd);
14529 bail_clear_intr:
14530         clean_up_interrupts(dd);
14531 bail_cleanup:
14532         hfi1_pcie_ddcleanup(dd);
14533 bail_free:
14534         hfi1_free_devdata(dd);
14535         dd = ERR_PTR(ret);
14536 bail:
14537         return dd;
14538 }
14539
14540 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
14541                         u32 dw_len)
14542 {
14543         u32 delta_cycles;
14544         u32 current_egress_rate = ppd->current_egress_rate;
14545         /* rates here are in units of 10^6 bits/sec */
14546
14547         if (desired_egress_rate == -1)
14548                 return 0; /* shouldn't happen */
14549
14550         if (desired_egress_rate >= current_egress_rate)
14551                 return 0; /* we can't help go faster, only slower */
14552
14553         delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
14554                         egress_cycles(dw_len * 4, current_egress_rate);
14555
14556         return (u16)delta_cycles;
14557 }
14558
14559 /**
14560  * create_pbc - build a pbc for transmission
14561  * @flags: special case flags or-ed in built pbc
14562  * @srate: static rate
14563  * @vl: vl
14564  * @dwlen: dword length (header words + data words + pbc words)
14565  *
14566  * Create a PBC with the given flags, rate, VL, and length.
14567  *
14568  * NOTE: The PBC created will not insert any HCRC - all callers but one are
14569  * for verbs, which does not use this PSM feature.  The lone other caller
14570  * is for the diagnostic interface which calls this if the user does not
14571  * supply their own PBC.
14572  */
14573 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
14574                u32 dw_len)
14575 {
14576         u64 pbc, delay = 0;
14577
14578         if (unlikely(srate_mbs))
14579                 delay = delay_cycles(ppd, srate_mbs, dw_len);
14580
14581         pbc = flags
14582                 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
14583                 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
14584                 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
14585                 | (dw_len & PBC_LENGTH_DWS_MASK)
14586                         << PBC_LENGTH_DWS_SHIFT;
14587
14588         return pbc;
14589 }
14590
14591 #define SBUS_THERMAL    0x4f
14592 #define SBUS_THERM_MONITOR_MODE 0x1
14593
14594 #define THERM_FAILURE(dev, ret, reason) \
14595         dd_dev_err((dd),                                                \
14596                    "Thermal sensor initialization failed: %s (%d)\n",   \
14597                    (reason), (ret))
14598
14599 /*
14600  * Initialize the thermal sensor.
14601  *
14602  * After initialization, enable polling of thermal sensor through
14603  * SBus interface. In order for this to work, the SBus Master
14604  * firmware has to be loaded due to the fact that the HW polling
14605  * logic uses SBus interrupts, which are not supported with
14606  * default firmware. Otherwise, no data will be returned through
14607  * the ASIC_STS_THERM CSR.
14608  */
14609 static int thermal_init(struct hfi1_devdata *dd)
14610 {
14611         int ret = 0;
14612
14613         if (dd->icode != ICODE_RTL_SILICON ||
14614             check_chip_resource(dd, CR_THERM_INIT, NULL))
14615                 return ret;
14616
14617         ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
14618         if (ret) {
14619                 THERM_FAILURE(dd, ret, "Acquire SBus");
14620                 return ret;
14621         }
14622
14623         dd_dev_info(dd, "Initializing thermal sensor\n");
14624         /* Disable polling of thermal readings */
14625         write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
14626         msleep(100);
14627         /* Thermal Sensor Initialization */
14628         /*    Step 1: Reset the Thermal SBus Receiver */
14629         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14630                                 RESET_SBUS_RECEIVER, 0);
14631         if (ret) {
14632                 THERM_FAILURE(dd, ret, "Bus Reset");
14633                 goto done;
14634         }
14635         /*    Step 2: Set Reset bit in Thermal block */
14636         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14637                                 WRITE_SBUS_RECEIVER, 0x1);
14638         if (ret) {
14639                 THERM_FAILURE(dd, ret, "Therm Block Reset");
14640                 goto done;
14641         }
14642         /*    Step 3: Write clock divider value (100MHz -> 2MHz) */
14643         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
14644                                 WRITE_SBUS_RECEIVER, 0x32);
14645         if (ret) {
14646                 THERM_FAILURE(dd, ret, "Write Clock Div");
14647                 goto done;
14648         }
14649         /*    Step 4: Select temperature mode */
14650         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
14651                                 WRITE_SBUS_RECEIVER,
14652                                 SBUS_THERM_MONITOR_MODE);
14653         if (ret) {
14654                 THERM_FAILURE(dd, ret, "Write Mode Sel");
14655                 goto done;
14656         }
14657         /*    Step 5: De-assert block reset and start conversion */
14658         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
14659                                 WRITE_SBUS_RECEIVER, 0x2);
14660         if (ret) {
14661                 THERM_FAILURE(dd, ret, "Write Reset Deassert");
14662                 goto done;
14663         }
14664         /*    Step 5.1: Wait for first conversion (21.5ms per spec) */
14665         msleep(22);
14666
14667         /* Enable polling of thermal readings */
14668         write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
14669
14670         /* Set initialized flag */
14671         ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
14672         if (ret)
14673                 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
14674
14675 done:
14676         release_chip_resource(dd, CR_SBUS);
14677         return ret;
14678 }
14679
14680 static void handle_temp_err(struct hfi1_devdata *dd)
14681 {
14682         struct hfi1_pportdata *ppd = &dd->pport[0];
14683         /*
14684          * Thermal Critical Interrupt
14685          * Put the device into forced freeze mode, take link down to
14686          * offline, and put DC into reset.
14687          */
14688         dd_dev_emerg(dd,
14689                      "Critical temperature reached! Forcing device into freeze mode!\n");
14690         dd->flags |= HFI1_FORCED_FREEZE;
14691         start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
14692         /*
14693          * Shut DC down as much and as quickly as possible.
14694          *
14695          * Step 1: Take the link down to OFFLINE. This will cause the
14696          *         8051 to put the Serdes in reset. However, we don't want to
14697          *         go through the entire link state machine since we want to
14698          *         shutdown ASAP. Furthermore, this is not a graceful shutdown
14699          *         but rather an attempt to save the chip.
14700          *         Code below is almost the same as quiet_serdes() but avoids
14701          *         all the extra work and the sleeps.
14702          */
14703         ppd->driver_link_ready = 0;
14704         ppd->link_enabled = 0;
14705         set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
14706                                 PLS_OFFLINE);
14707         /*
14708          * Step 2: Shutdown LCB and 8051
14709          *         After shutdown, do not restore DC_CFG_RESET value.
14710          */
14711         dc_shutdown(dd);
14712 }