drivers/net/ethernet/amazon/ena/ena_com.c

   1 /*
   2  * Copyright 2015 Amazon.com, Inc. or its affiliates.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include "ena_com.h"
  34
  35 /*****************************************************************************/
  36 /*****************************************************************************/
  37
  38 /* Timeout in micro-sec */
  39 #define ADMIN_CMD_TIMEOUT_US (1000000)
  40
  41 #define ENA_ASYNC_QUEUE_DEPTH 4
  42 #define ENA_ADMIN_QUEUE_DEPTH 32
  43
  44 #define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
  45                 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
  46                 | (ENA_COMMON_SPEC_VERSION_MINOR))
  47
  48 #define ENA_CTRL_MAJOR          0
  49 #define ENA_CTRL_MINOR          0
  50 #define ENA_CTRL_SUB_MINOR      1
  51
  52 #define MIN_ENA_CTRL_VER \
  53         (((ENA_CTRL_MAJOR) << \
  54         (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
  55         ((ENA_CTRL_MINOR) << \
  56         (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
  57         (ENA_CTRL_SUB_MINOR))
  58
  59 #define ENA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))
  60 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) >> 32))
  61
  62 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
  63
  64 /*****************************************************************************/
  65 /*****************************************************************************/
  66 /*****************************************************************************/
  67
  68 enum ena_cmd_status {
  69         ENA_CMD_SUBMITTED,
  70         ENA_CMD_COMPLETED,
  71         /* Abort - canceled by the driver */
  72         ENA_CMD_ABORTED,
  73 };
  74
  75 struct ena_comp_ctx {
  76         struct completion wait_event;
  77         struct ena_admin_acq_entry *user_cqe;
  78         u32 comp_size;
  79         enum ena_cmd_status status;
  80         /* status from the device */
  81         u8 comp_status;
  82         u8 cmd_opcode;
  83         bool occupied;
  84 };
  85
  86 struct ena_com_stats_ctx {
  87         struct ena_admin_aq_get_stats_cmd get_cmd;
  88         struct ena_admin_acq_get_stats_resp get_resp;
  89 };
  90
  91 static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
  92                                        struct ena_common_mem_addr *ena_addr,
  93                                        dma_addr_t addr)
  94 {
  95         if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
  96                 pr_err("dma address has more bits that the device supports\n");
  97                 return -EINVAL;
  98         }
  99
 100         ena_addr->mem_addr_low = (u32)addr;
 101         ena_addr->mem_addr_high = (u64)addr >> 32;
 102
 103         return 0;
 104 }
 105
 106 static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
 107 {
 108         struct ena_com_admin_sq *sq = &queue->sq;
 109         u16 size = ADMIN_SQ_SIZE(queue->q_depth);
 110
 111         sq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &sq->dma_addr,
 112                                           GFP_KERNEL);
 113
 114         if (!sq->entries) {
 115                 pr_err("memory allocation failed");
 116                 return -ENOMEM;
 117         }
 118
 119         sq->head = 0;
 120         sq->tail = 0;
 121         sq->phase = 1;
 122
 123         sq->db_addr = NULL;
 124
 125         return 0;
 126 }
 127
 128 static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
 129 {
 130         struct ena_com_admin_cq *cq = &queue->cq;
 131         u16 size = ADMIN_CQ_SIZE(queue->q_depth);
 132
 133         cq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &cq->dma_addr,
 134                                           GFP_KERNEL);
 135
 136         if (!cq->entries) {
 137                 pr_err("memory allocation failed");
 138                 return -ENOMEM;
 139         }
 140
 141         cq->head = 0;
 142         cq->phase = 1;
 143
 144         return 0;
 145 }
 146
 147 static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
 148                                    struct ena_aenq_handlers *aenq_handlers)
 149 {
 150         struct ena_com_aenq *aenq = &dev->aenq;
 151         u32 addr_low, addr_high, aenq_caps;
 152         u16 size;
 153
 154         dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
 155         size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
 156         aenq->entries = dma_zalloc_coherent(dev->dmadev, size, &aenq->dma_addr,
 157                                             GFP_KERNEL);
 158
 159         if (!aenq->entries) {
 160                 pr_err("memory allocation failed");
 161                 return -ENOMEM;
 162         }
 163
 164         aenq->head = aenq->q_depth;
 165         aenq->phase = 1;
 166
 167         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
 168         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
 169
 170         writel(addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
 171         writel(addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
 172
 173         aenq_caps = 0;
 174         aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
 175         aenq_caps |= (sizeof(struct ena_admin_aenq_entry)
 176                       << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
 177                      ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
 178         writel(aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
 179
 180         if (unlikely(!aenq_handlers)) {
 181                 pr_err("aenq handlers pointer is NULL\n");
 182                 return -EINVAL;
 183         }
 184
 185         aenq->aenq_handlers = aenq_handlers;
 186
 187         return 0;
 188 }
 189
 190 static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
 191                                      struct ena_comp_ctx *comp_ctx)
 192 {
 193         comp_ctx->occupied = false;
 194         atomic_dec(&queue->outstanding_cmds);
 195 }
 196
 197 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
 198                                           u16 command_id, bool capture)
 199 {
 200         if (unlikely(command_id >= queue->q_depth)) {
 201                 pr_err("command id is larger than the queue size. cmd_id: %u queue size %d\n",
 202                        command_id, queue->q_depth);
 203                 return NULL;
 204         }
 205
 206         if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
 207                 pr_err("Completion context is occupied\n");
 208                 return NULL;
 209         }
 210
 211         if (capture) {
 212                 atomic_inc(&queue->outstanding_cmds);
 213                 queue->comp_ctx[command_id].occupied = true;
 214         }
 215
 216         return &queue->comp_ctx[command_id];
 217 }
 218
 219 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 220                                                        struct ena_admin_aq_entry *cmd,
 221                                                        size_t cmd_size_in_bytes,
 222                                                        struct ena_admin_acq_entry *comp,
 223                                                        size_t comp_size_in_bytes)
 224 {
 225         struct ena_comp_ctx *comp_ctx;
 226         u16 tail_masked, cmd_id;
 227         u16 queue_size_mask;
 228         u16 cnt;
 229
 230         queue_size_mask = admin_queue->q_depth - 1;
 231
 232         tail_masked = admin_queue->sq.tail & queue_size_mask;
 233
 234         /* In case of queue FULL */
 235         cnt = admin_queue->sq.tail - admin_queue->sq.head;
 236         if (cnt >= admin_queue->q_depth) {
 237                 pr_debug("admin queue is FULL (tail %d head %d depth: %d)\n",
 238                          admin_queue->sq.tail, admin_queue->sq.head,
 239                          admin_queue->q_depth);
 240                 admin_queue->stats.out_of_space++;
 241                 return ERR_PTR(-ENOSPC);
 242         }
 243
 244         cmd_id = admin_queue->curr_cmd_id;
 245
 246         cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
 247                 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
 248
 249         cmd->aq_common_descriptor.command_id |= cmd_id &
 250                 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 251
 252         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
 253         if (unlikely(!comp_ctx))
 254                 return ERR_PTR(-EINVAL);
 255
 256         comp_ctx->status = ENA_CMD_SUBMITTED;
 257         comp_ctx->comp_size = (u32)comp_size_in_bytes;
 258         comp_ctx->user_cqe = comp;
 259         comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
 260
 261         reinit_completion(&comp_ctx->wait_event);
 262
 263         memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
 264
 265         admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
 266                 queue_size_mask;
 267
 268         admin_queue->sq.tail++;
 269         admin_queue->stats.submitted_cmd++;
 270
 271         if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
 272                 admin_queue->sq.phase = !admin_queue->sq.phase;
 273
 274         writel(admin_queue->sq.tail, admin_queue->sq.db_addr);
 275
 276         return comp_ctx;
 277 }
 278
 279 static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
 280 {
 281         size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
 282         struct ena_comp_ctx *comp_ctx;
 283         u16 i;
 284
 285         queue->comp_ctx = devm_kzalloc(queue->q_dmadev, size, GFP_KERNEL);
 286         if (unlikely(!queue->comp_ctx)) {
 287                 pr_err("memory allocation failed");
 288                 return -ENOMEM;
 289         }
 290
 291         for (i = 0; i < queue->q_depth; i++) {
 292                 comp_ctx = get_comp_ctxt(queue, i, false);
 293                 if (comp_ctx)
 294                         init_completion(&comp_ctx->wait_event);
 295         }
 296
 297         return 0;
 298 }
 299
 300 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 301                                                      struct ena_admin_aq_entry *cmd,
 302                                                      size_t cmd_size_in_bytes,
 303                                                      struct ena_admin_acq_entry *comp,
 304                                                      size_t comp_size_in_bytes)
 305 {
 306         unsigned long flags;
 307         struct ena_comp_ctx *comp_ctx;
 308
 309         spin_lock_irqsave(&admin_queue->q_lock, flags);
 310         if (unlikely(!admin_queue->running_state)) {
 311                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 312                 return ERR_PTR(-ENODEV);
 313         }
 314         comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
 315                                               cmd_size_in_bytes,
 316                                               comp,
 317                                               comp_size_in_bytes);
 318         if (unlikely(IS_ERR(comp_ctx)))
 319                 admin_queue->running_state = false;
 320         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 321
 322         return comp_ctx;
 323 }
 324
 325 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 326                               struct ena_com_create_io_ctx *ctx,
 327                               struct ena_com_io_sq *io_sq)
 328 {
 329         size_t size;
 330         int dev_node = 0;
 331
 332         memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
 333
 334         io_sq->desc_entry_size =
 335                 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
 336                 sizeof(struct ena_eth_io_tx_desc) :
 337                 sizeof(struct ena_eth_io_rx_desc);
 338
 339         size = io_sq->desc_entry_size * io_sq->q_depth;
 340
 341         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
 342                 dev_node = dev_to_node(ena_dev->dmadev);
 343                 set_dev_node(ena_dev->dmadev, ctx->numa_node);
 344                 io_sq->desc_addr.virt_addr =
 345                         dma_zalloc_coherent(ena_dev->dmadev, size,
 346                                             &io_sq->desc_addr.phys_addr,
 347                                             GFP_KERNEL);
 348                 set_dev_node(ena_dev->dmadev, dev_node);
 349                 if (!io_sq->desc_addr.virt_addr) {
 350                         io_sq->desc_addr.virt_addr =
 351                                 dma_zalloc_coherent(ena_dev->dmadev, size,
 352                                                     &io_sq->desc_addr.phys_addr,
 353                                                     GFP_KERNEL);
 354                 }
 355         } else {
 356                 dev_node = dev_to_node(ena_dev->dmadev);
 357                 set_dev_node(ena_dev->dmadev, ctx->numa_node);
 358                 io_sq->desc_addr.virt_addr =
 359                         devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
 360                 set_dev_node(ena_dev->dmadev, dev_node);
 361                 if (!io_sq->desc_addr.virt_addr) {
 362                         io_sq->desc_addr.virt_addr =
 363                                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
 364                 }
 365         }
 366
 367         if (!io_sq->desc_addr.virt_addr) {
 368                 pr_err("memory allocation failed");
 369                 return -ENOMEM;
 370         }
 371
 372         io_sq->tail = 0;
 373         io_sq->next_to_comp = 0;
 374         io_sq->phase = 1;
 375
 376         return 0;
 377 }
 378
 379 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
 380                               struct ena_com_create_io_ctx *ctx,
 381                               struct ena_com_io_cq *io_cq)
 382 {
 383         size_t size;
 384         int prev_node = 0;
 385
 386         memset(&io_cq->cdesc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
 387
 388         /* Use the basic completion descriptor for Rx */
 389         io_cq->cdesc_entry_size_in_bytes =
 390                 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
 391                 sizeof(struct ena_eth_io_tx_cdesc) :
 392                 sizeof(struct ena_eth_io_rx_cdesc_base);
 393
 394         size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 395
 396         prev_node = dev_to_node(ena_dev->dmadev);
 397         set_dev_node(ena_dev->dmadev, ctx->numa_node);
 398         io_cq->cdesc_addr.virt_addr =
 399                 dma_zalloc_coherent(ena_dev->dmadev, size,
 400                                     &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
 401         set_dev_node(ena_dev->dmadev, prev_node);
 402         if (!io_cq->cdesc_addr.virt_addr) {
 403                 io_cq->cdesc_addr.virt_addr =
 404                         dma_zalloc_coherent(ena_dev->dmadev, size,
 405                                             &io_cq->cdesc_addr.phys_addr,
 406                                             GFP_KERNEL);
 407         }
 408
 409         if (!io_cq->cdesc_addr.virt_addr) {
 410                 pr_err("memory allocation failed");
 411                 return -ENOMEM;
 412         }
 413
 414         io_cq->phase = 1;
 415         io_cq->head = 0;
 416
 417         return 0;
 418 }
 419
 420 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
 421                                                    struct ena_admin_acq_entry *cqe)
 422 {
 423         struct ena_comp_ctx *comp_ctx;
 424         u16 cmd_id;
 425
 426         cmd_id = cqe->acq_common_descriptor.command &
 427                 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
 428
 429         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
 430         if (unlikely(!comp_ctx)) {
 431                 pr_err("comp_ctx is NULL. Changing the admin queue running state\n");
 432                 admin_queue->running_state = false;
 433                 return;
 434         }
 435
 436         comp_ctx->status = ENA_CMD_COMPLETED;
 437         comp_ctx->comp_status = cqe->acq_common_descriptor.status;
 438
 439         if (comp_ctx->user_cqe)
 440                 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
 441
 442         if (!admin_queue->polling)
 443                 complete(&comp_ctx->wait_event);
 444 }
 445
 446 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
 447 {
 448         struct ena_admin_acq_entry *cqe = NULL;
 449         u16 comp_num = 0;
 450         u16 head_masked;
 451         u8 phase;
 452
 453         head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
 454         phase = admin_queue->cq.phase;
 455
 456         cqe = &admin_queue->cq.entries[head_masked];
 457
 458         /* Go over all the completions */
 459         while ((cqe->acq_common_descriptor.flags &
 460                         ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
 461                 /* Do not read the rest of the completion entry before the
 462                  * phase bit was validated
 463                  */
 464                 rmb();
 465                 ena_com_handle_single_admin_completion(admin_queue, cqe);
 466
 467                 head_masked++;
 468                 comp_num++;
 469                 if (unlikely(head_masked == admin_queue->q_depth)) {
 470                         head_masked = 0;
 471                         phase = !phase;
 472                 }
 473
 474                 cqe = &admin_queue->cq.entries[head_masked];
 475         }
 476
 477         admin_queue->cq.head += comp_num;
 478         admin_queue->cq.phase = phase;
 479         admin_queue->sq.head += comp_num;
 480         admin_queue->stats.completed_cmd += comp_num;
 481 }
 482
 483 static int ena_com_comp_status_to_errno(u8 comp_status)
 484 {
 485         if (unlikely(comp_status != 0))
 486                 pr_err("admin command failed[%u]\n", comp_status);
 487
 488         if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
 489                 return -EINVAL;
 490
 491         switch (comp_status) {
 492         case ENA_ADMIN_SUCCESS:
 493                 return 0;
 494         case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
 495                 return -ENOMEM;
 496         case ENA_ADMIN_UNSUPPORTED_OPCODE:
 497                 return -EPERM;
 498         case ENA_ADMIN_BAD_OPCODE:
 499         case ENA_ADMIN_MALFORMED_REQUEST:
 500         case ENA_ADMIN_ILLEGAL_PARAMETER:
 501         case ENA_ADMIN_UNKNOWN_ERROR:
 502                 return -EINVAL;
 503         }
 504
 505         return 0;
 506 }
 507
 508 static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
 509                                                      struct ena_com_admin_queue *admin_queue)
 510 {
 511         unsigned long flags;
 512         u32 start_time;
 513         int ret;
 514
 515         start_time = ((u32)jiffies_to_usecs(jiffies));
 516
 517         while (comp_ctx->status == ENA_CMD_SUBMITTED) {
 518                 if ((((u32)jiffies_to_usecs(jiffies)) - start_time) >
 519                     ADMIN_CMD_TIMEOUT_US) {
 520                         pr_err("Wait for completion (polling) timeout\n");
 521                         /* ENA didn't have any completion */
 522                         spin_lock_irqsave(&admin_queue->q_lock, flags);
 523                         admin_queue->stats.no_completion++;
 524                         admin_queue->running_state = false;
 525                         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 526
 527                         ret = -ETIME;
 528                         goto err;
 529                 }
 530
 531                 spin_lock_irqsave(&admin_queue->q_lock, flags);
 532                 ena_com_handle_admin_completion(admin_queue);
 533                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 534
 535                 msleep(100);
 536         }
 537
 538         if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
 539                 pr_err("Command was aborted\n");
 540                 spin_lock_irqsave(&admin_queue->q_lock, flags);
 541                 admin_queue->stats.aborted_cmd++;
 542                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 543                 ret = -ENODEV;
 544                 goto err;
 545         }
 546
 547         WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n",
 548              comp_ctx->status);
 549
 550         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 551 err:
 552         comp_ctxt_release(admin_queue, comp_ctx);
 553         return ret;
 554 }
 555
 556 static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
 557                                                         struct ena_com_admin_queue *admin_queue)
 558 {
 559         unsigned long flags;
 560         int ret;
 561
 562         wait_for_completion_timeout(&comp_ctx->wait_event,
 563                                     usecs_to_jiffies(ADMIN_CMD_TIMEOUT_US));
 564
 565         /* In case the command wasn't completed find out the root cause.
 566          * There might be 2 kinds of errors
 567          * 1) No completion (timeout reached)
 568          * 2) There is completion but the device didn't get any msi-x interrupt.
 569          */
 570         if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
 571                 spin_lock_irqsave(&admin_queue->q_lock, flags);
 572                 ena_com_handle_admin_completion(admin_queue);
 573                 admin_queue->stats.no_completion++;
 574                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 575
 576                 if (comp_ctx->status == ENA_CMD_COMPLETED)
 577                         pr_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
 578                                comp_ctx->cmd_opcode);
 579                 else
 580                         pr_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
 581                                comp_ctx->cmd_opcode, comp_ctx->status);
 582
 583                 admin_queue->running_state = false;
 584                 ret = -ETIME;
 585                 goto err;
 586         }
 587
 588         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 589 err:
 590         comp_ctxt_release(admin_queue, comp_ctx);
 591         return ret;
 592 }
 593
 594 /* This method read the hardware device register through posting writes
 595  * and waiting for response
 596  * On timeout the function will return ENA_MMIO_READ_TIMEOUT
 597  */
 598 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 599 {
 600         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
 601         volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
 602                 mmio_read->read_resp;
 603         u32 mmio_read_reg, ret;
 604         unsigned long flags;
 605         int i;
 606
 607         might_sleep();
 608
 609         /* If readless is disabled, perform regular read */
 610         if (!mmio_read->readless_supported)
 611                 return readl(ena_dev->reg_bar + offset);
 612
 613         spin_lock_irqsave(&mmio_read->lock, flags);
 614         mmio_read->seq_num++;
 615
 616         read_resp->req_id = mmio_read->seq_num + 0xDEAD;
 617         mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
 618                         ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
 619         mmio_read_reg |= mmio_read->seq_num &
 620                         ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
 621
 622         /* make sure read_resp->req_id get updated before the hw can write
 623          * there
 624          */
 625         wmb();
 626
 627         writel(mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
 628
 629         for (i = 0; i < ENA_REG_READ_TIMEOUT; i++) {
 630                 if (read_resp->req_id == mmio_read->seq_num)
 631                         break;
 632
 633                 udelay(1);
 634         }
 635
 636         if (unlikely(i == ENA_REG_READ_TIMEOUT)) {
 637                 pr_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
 638                        mmio_read->seq_num, offset, read_resp->req_id,
 639                        read_resp->reg_off);
 640                 ret = ENA_MMIO_READ_TIMEOUT;
 641                 goto err;
 642         }
 643
 644         if (read_resp->reg_off != offset) {
 645                 pr_err("Read failure: wrong offset provided");
 646                 ret = ENA_MMIO_READ_TIMEOUT;
 647         } else {
 648                 ret = read_resp->reg_val;
 649         }
 650 err:
 651         spin_unlock_irqrestore(&mmio_read->lock, flags);
 652
 653         return ret;
 654 }
 655
 656 /* There are two types to wait for completion.
 657  * Polling mode - wait until the completion is available.
 658  * Async mode - wait on wait queue until the completion is ready
 659  * (or the timeout expired).
 660  * It is expected that the IRQ called ena_com_handle_admin_completion
 661  * to mark the completions.
 662  */
 663 static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
 664                                              struct ena_com_admin_queue *admin_queue)
 665 {
 666         if (admin_queue->polling)
 667                 return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
 668                                                                  admin_queue);
 669
 670         return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
 671                                                             admin_queue);
 672 }
 673
 674 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 675                                  struct ena_com_io_sq *io_sq)
 676 {
 677         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 678         struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
 679         struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
 680         u8 direction;
 681         int ret;
 682
 683         memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
 684
 685         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 686                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
 687         else
 688                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
 689
 690         destroy_cmd.sq.sq_identity |= (direction <<
 691                 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
 692                 ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
 693
 694         destroy_cmd.sq.sq_idx = io_sq->idx;
 695         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
 696
 697         ret = ena_com_execute_admin_command(admin_queue,
 698                                             (struct ena_admin_aq_entry *)&destroy_cmd,
 699                                             sizeof(destroy_cmd),
 700                                             (struct ena_admin_acq_entry *)&destroy_resp,
 701                                             sizeof(destroy_resp));
 702
 703         if (unlikely(ret && (ret != -ENODEV)))
 704                 pr_err("failed to destroy io sq error: %d\n", ret);
 705
 706         return ret;
 707 }
 708
 709 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
 710                                   struct ena_com_io_sq *io_sq,
 711                                   struct ena_com_io_cq *io_cq)
 712 {
 713         size_t size;
 714
 715         if (io_cq->cdesc_addr.virt_addr) {
 716                 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 717
 718                 dma_free_coherent(ena_dev->dmadev, size,
 719                                   io_cq->cdesc_addr.virt_addr,
 720                                   io_cq->cdesc_addr.phys_addr);
 721
 722                 io_cq->cdesc_addr.virt_addr = NULL;
 723         }
 724
 725         if (io_sq->desc_addr.virt_addr) {
 726                 size = io_sq->desc_entry_size * io_sq->q_depth;
 727
 728                 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 729                         dma_free_coherent(ena_dev->dmadev, size,
 730                                           io_sq->desc_addr.virt_addr,
 731                                           io_sq->desc_addr.phys_addr);
 732                 else
 733                         devm_kfree(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
 734
 735                 io_sq->desc_addr.virt_addr = NULL;
 736         }
 737 }
 738
 739 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
 740                                 u16 exp_state)
 741 {
 742         u32 val, i;
 743
 744         for (i = 0; i < timeout; i++) {
 745                 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
 746
 747                 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
 748                         pr_err("Reg read timeout occurred\n");
 749                         return -ETIME;
 750                 }
 751
 752                 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
 753                         exp_state)
 754                         return 0;
 755
 756                 /* The resolution of the timeout is 100ms */
 757                 msleep(100);
 758         }
 759
 760         return -ETIME;
 761 }
 762
 763 static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
 764                                                enum ena_admin_aq_feature_id feature_id)
 765 {
 766         u32 feature_mask = 1 << feature_id;
 767
 768         /* Device attributes is always supported */
 769         if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
 770             !(ena_dev->supported_features & feature_mask))
 771                 return false;
 772
 773         return true;
 774 }
 775
 776 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
 777                                   struct ena_admin_get_feat_resp *get_resp,
 778                                   enum ena_admin_aq_feature_id feature_id,
 779                                   dma_addr_t control_buf_dma_addr,
 780                                   u32 control_buff_size)
 781 {
 782         struct ena_com_admin_queue *admin_queue;
 783         struct ena_admin_get_feat_cmd get_cmd;
 784         int ret;
 785
 786         if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
 787                 pr_info("Feature %d isn't supported\n", feature_id);
 788                 return -EPERM;
 789         }
 790
 791         memset(&get_cmd, 0x0, sizeof(get_cmd));
 792         admin_queue = &ena_dev->admin_queue;
 793
 794         get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
 795
 796         if (control_buff_size)
 797                 get_cmd.aq_common_descriptor.flags =
 798                         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
 799         else
 800                 get_cmd.aq_common_descriptor.flags = 0;
 801
 802         ret = ena_com_mem_addr_set(ena_dev,
 803                                    &get_cmd.control_buffer.address,
 804                                    control_buf_dma_addr);
 805         if (unlikely(ret)) {
 806                 pr_err("memory address set failed\n");
 807                 return ret;
 808         }
 809
 810         get_cmd.control_buffer.length = control_buff_size;
 811
 812         get_cmd.feat_common.feature_id = feature_id;
 813
 814         ret = ena_com_execute_admin_command(admin_queue,
 815                                             (struct ena_admin_aq_entry *)
 816                                             &get_cmd,
 817                                             sizeof(get_cmd),
 818                                             (struct ena_admin_acq_entry *)
 819                                             get_resp,
 820                                             sizeof(*get_resp));
 821
 822         if (unlikely(ret))
 823                 pr_err("Failed to submit get_feature command %d error: %d\n",
 824                        feature_id, ret);
 825
 826         return ret;
 827 }
 828
 829 static int ena_com_get_feature(struct ena_com_dev *ena_dev,
 830                                struct ena_admin_get_feat_resp *get_resp,
 831                                enum ena_admin_aq_feature_id feature_id)
 832 {
 833         return ena_com_get_feature_ex(ena_dev,
 834                                       get_resp,
 835                                       feature_id,
 836                                       0,
 837                                       0);
 838 }
 839
 840 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
 841 {
 842         struct ena_rss *rss = &ena_dev->rss;
 843
 844         rss->hash_key =
 845                 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
 846                                     &rss->hash_key_dma_addr, GFP_KERNEL);
 847
 848         if (unlikely(!rss->hash_key))
 849                 return -ENOMEM;
 850
 851         return 0;
 852 }
 853
 854 static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
 855 {
 856         struct ena_rss *rss = &ena_dev->rss;
 857
 858         if (rss->hash_key)
 859                 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
 860                                   rss->hash_key, rss->hash_key_dma_addr);
 861         rss->hash_key = NULL;
 862 }
 863
 864 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
 865 {
 866         struct ena_rss *rss = &ena_dev->rss;
 867
 868         rss->hash_ctrl =
 869                 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
 870                                     &rss->hash_ctrl_dma_addr, GFP_KERNEL);
 871
 872         if (unlikely(!rss->hash_ctrl))
 873                 return -ENOMEM;
 874
 875         return 0;
 876 }
 877
 878 static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
 879 {
 880         struct ena_rss *rss = &ena_dev->rss;
 881
 882         if (rss->hash_ctrl)
 883                 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
 884                                   rss->hash_ctrl, rss->hash_ctrl_dma_addr);
 885         rss->hash_ctrl = NULL;
 886 }
 887
 888 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
 889                                            u16 log_size)
 890 {
 891         struct ena_rss *rss = &ena_dev->rss;
 892         struct ena_admin_get_feat_resp get_resp;
 893         size_t tbl_size;
 894         int ret;
 895
 896         ret = ena_com_get_feature(ena_dev, &get_resp,
 897                                   ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
 898         if (unlikely(ret))
 899                 return ret;
 900
 901         if ((get_resp.u.ind_table.min_size > log_size) ||
 902             (get_resp.u.ind_table.max_size < log_size)) {
 903                 pr_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
 904                        1 << log_size, 1 << get_resp.u.ind_table.min_size,
 905                        1 << get_resp.u.ind_table.max_size);
 906                 return -EINVAL;
 907         }
 908
 909         tbl_size = (1ULL << log_size) *
 910                 sizeof(struct ena_admin_rss_ind_table_entry);
 911
 912         rss->rss_ind_tbl =
 913                 dma_zalloc_coherent(ena_dev->dmadev, tbl_size,
 914                                     &rss->rss_ind_tbl_dma_addr, GFP_KERNEL);
 915         if (unlikely(!rss->rss_ind_tbl))
 916                 goto mem_err1;
 917
 918         tbl_size = (1ULL << log_size) * sizeof(u16);
 919         rss->host_rss_ind_tbl =
 920                 devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL);
 921         if (unlikely(!rss->host_rss_ind_tbl))
 922                 goto mem_err2;
 923
 924         rss->tbl_log_size = log_size;
 925
 926         return 0;
 927
 928 mem_err2:
 929         tbl_size = (1ULL << log_size) *
 930                 sizeof(struct ena_admin_rss_ind_table_entry);
 931
 932         dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
 933                           rss->rss_ind_tbl_dma_addr);
 934         rss->rss_ind_tbl = NULL;
 935 mem_err1:
 936         rss->tbl_log_size = 0;
 937         return -ENOMEM;
 938 }
 939
 940 static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
 941 {
 942         struct ena_rss *rss = &ena_dev->rss;
 943         size_t tbl_size = (1ULL << rss->tbl_log_size) *
 944                 sizeof(struct ena_admin_rss_ind_table_entry);
 945
 946         if (rss->rss_ind_tbl)
 947                 dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
 948                                   rss->rss_ind_tbl_dma_addr);
 949         rss->rss_ind_tbl = NULL;
 950
 951         if (rss->host_rss_ind_tbl)
 952                 devm_kfree(ena_dev->dmadev, rss->host_rss_ind_tbl);
 953         rss->host_rss_ind_tbl = NULL;
 954 }
 955
 956 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
 957                                 struct ena_com_io_sq *io_sq, u16 cq_idx)
 958 {
 959         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 960         struct ena_admin_aq_create_sq_cmd create_cmd;
 961         struct ena_admin_acq_create_sq_resp_desc cmd_completion;
 962         u8 direction;
 963         int ret;
 964
 965         memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_sq_cmd));
 966
 967         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
 968
 969         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 970                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
 971         else
 972                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
 973
 974         create_cmd.sq_identity |= (direction <<
 975                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
 976                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
 977
 978         create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
 979                 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
 980
 981         create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
 982                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
 983                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
 984
 985         create_cmd.sq_caps_3 |=
 986                 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
 987
 988         create_cmd.cq_idx = cq_idx;
 989         create_cmd.sq_depth = io_sq->q_depth;
 990
 991         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
 992                 ret = ena_com_mem_addr_set(ena_dev,
 993                                            &create_cmd.sq_ba,
 994                                            io_sq->desc_addr.phys_addr);
 995                 if (unlikely(ret)) {
 996                         pr_err("memory address set failed\n");
 997                         return ret;
 998                 }
 999         }
1000
1001         ret = ena_com_execute_admin_command(admin_queue,
1002                                             (struct ena_admin_aq_entry *)&create_cmd,
1003                                             sizeof(create_cmd),
1004                                             (struct ena_admin_acq_entry *)&cmd_completion,
1005                                             sizeof(cmd_completion));
1006         if (unlikely(ret)) {
1007                 pr_err("Failed to create IO SQ. error: %d\n", ret);
1008                 return ret;
1009         }
1010
1011         io_sq->idx = cmd_completion.sq_idx;
1012
1013         io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1014                 (uintptr_t)cmd_completion.sq_doorbell_offset);
1015
1016         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1017                 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
1018                                 + cmd_completion.llq_headers_offset);
1019
1020                 io_sq->desc_addr.pbuf_dev_addr =
1021                         (u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
1022                         cmd_completion.llq_descriptors_offset);
1023         }
1024
1025         pr_debug("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
1026
1027         return ret;
1028 }
1029
1030 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
1031 {
1032         struct ena_rss *rss = &ena_dev->rss;
1033         struct ena_com_io_sq *io_sq;
1034         u16 qid;
1035         int i;
1036
1037         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1038                 qid = rss->host_rss_ind_tbl[i];
1039                 if (qid >= ENA_TOTAL_NUM_QUEUES)
1040                         return -EINVAL;
1041
1042                 io_sq = &ena_dev->io_sq_queues[qid];
1043
1044                 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
1045                         return -EINVAL;
1046
1047                 rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
1048         }
1049
1050         return 0;
1051 }
1052
1053 static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
1054 {
1055         u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 };
1056         struct ena_rss *rss = &ena_dev->rss;
1057         u8 idx;
1058         u16 i;
1059
1060         for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++)
1061                 dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i;
1062
1063         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1064                 if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES)
1065                         return -EINVAL;
1066                 idx = (u8)rss->rss_ind_tbl[i].cq_idx;
1067
1068                 if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES)
1069                         return -EINVAL;
1070
1071                 rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx];
1072         }
1073
1074         return 0;
1075 }
1076
1077 static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
1078 {
1079         size_t size;
1080
1081         size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
1082
1083         ena_dev->intr_moder_tbl =
1084                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
1085         if (!ena_dev->intr_moder_tbl)
1086                 return -ENOMEM;
1087
1088         ena_com_config_default_interrupt_moderation_table(ena_dev);
1089
1090         return 0;
1091 }
1092
1093 static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
1094                                                  u16 intr_delay_resolution)
1095 {
1096         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
1097         unsigned int i;
1098
1099         if (!intr_delay_resolution) {
1100                 pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
1101                 intr_delay_resolution = 1;
1102         }
1103         ena_dev->intr_delay_resolution = intr_delay_resolution;
1104
1105         /* update Rx */
1106         for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
1107                 intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
1108
1109         /* update Tx */
1110         ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
1111 }
1112
1113 /*****************************************************************************/
1114 /*******************************      API       ******************************/
1115 /*****************************************************************************/
1116
1117 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
1118                                   struct ena_admin_aq_entry *cmd,
1119                                   size_t cmd_size,
1120                                   struct ena_admin_acq_entry *comp,
1121                                   size_t comp_size)
1122 {
1123         struct ena_comp_ctx *comp_ctx;
1124         int ret;
1125
1126         comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
1127                                             comp, comp_size);
1128         if (unlikely(IS_ERR(comp_ctx))) {
1129                 pr_err("Failed to submit command [%ld]\n", PTR_ERR(comp_ctx));
1130                 return PTR_ERR(comp_ctx);
1131         }
1132
1133         ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
1134         if (unlikely(ret)) {
1135                 if (admin_queue->running_state)
1136                         pr_err("Failed to process command. ret = %d\n", ret);
1137                 else
1138                         pr_debug("Failed to process command. ret = %d\n", ret);
1139         }
1140         return ret;
1141 }
1142
1143 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
1144                          struct ena_com_io_cq *io_cq)
1145 {
1146         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1147         struct ena_admin_aq_create_cq_cmd create_cmd;
1148         struct ena_admin_acq_create_cq_resp_desc cmd_completion;
1149         int ret;
1150
1151         memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_cq_cmd));
1152
1153         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
1154
1155         create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
1156                 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
1157         create_cmd.cq_caps_1 |=
1158                 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
1159
1160         create_cmd.msix_vector = io_cq->msix_vector;
1161         create_cmd.cq_depth = io_cq->q_depth;
1162
1163         ret = ena_com_mem_addr_set(ena_dev,
1164                                    &create_cmd.cq_ba,
1165                                    io_cq->cdesc_addr.phys_addr);
1166         if (unlikely(ret)) {
1167                 pr_err("memory address set failed\n");
1168                 return ret;
1169         }
1170
1171         ret = ena_com_execute_admin_command(admin_queue,
1172                                             (struct ena_admin_aq_entry *)&create_cmd,
1173                                             sizeof(create_cmd),
1174                                             (struct ena_admin_acq_entry *)&cmd_completion,
1175                                             sizeof(cmd_completion));
1176         if (unlikely(ret)) {
1177                 pr_err("Failed to create IO CQ. error: %d\n", ret);
1178                 return ret;
1179         }
1180
1181         io_cq->idx = cmd_completion.cq_idx;
1182
1183         io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1184                 cmd_completion.cq_interrupt_unmask_register_offset);
1185
1186         if (cmd_completion.cq_head_db_register_offset)
1187                 io_cq->cq_head_db_reg =
1188                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1189                         cmd_completion.cq_head_db_register_offset);
1190
1191         if (cmd_completion.numa_node_register_offset)
1192                 io_cq->numa_node_cfg_reg =
1193                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1194                         cmd_completion.numa_node_register_offset);
1195
1196         pr_debug("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
1197
1198         return ret;
1199 }
1200
1201 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
1202                             struct ena_com_io_sq **io_sq,
1203                             struct ena_com_io_cq **io_cq)
1204 {
1205         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1206                 pr_err("Invalid queue number %d but the max is %d\n", qid,
1207                        ENA_TOTAL_NUM_QUEUES);
1208                 return -EINVAL;
1209         }
1210
1211         *io_sq = &ena_dev->io_sq_queues[qid];
1212         *io_cq = &ena_dev->io_cq_queues[qid];
1213
1214         return 0;
1215 }
1216
1217 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
1218 {
1219         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1220         struct ena_comp_ctx *comp_ctx;
1221         u16 i;
1222
1223         if (!admin_queue->comp_ctx)
1224                 return;
1225
1226         for (i = 0; i < admin_queue->q_depth; i++) {
1227                 comp_ctx = get_comp_ctxt(admin_queue, i, false);
1228                 if (unlikely(!comp_ctx))
1229                         break;
1230
1231                 comp_ctx->status = ENA_CMD_ABORTED;
1232
1233                 complete(&comp_ctx->wait_event);
1234         }
1235 }
1236
1237 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
1238 {
1239         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1240         unsigned long flags;
1241
1242         spin_lock_irqsave(&admin_queue->q_lock, flags);
1243         while (atomic_read(&admin_queue->outstanding_cmds) != 0) {
1244                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1245                 msleep(20);
1246                 spin_lock_irqsave(&admin_queue->q_lock, flags);
1247         }
1248         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1249 }
1250
1251 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
1252                           struct ena_com_io_cq *io_cq)
1253 {
1254         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1255         struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
1256         struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
1257         int ret;
1258
1259         memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
1260
1261         destroy_cmd.cq_idx = io_cq->idx;
1262         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
1263
1264         ret = ena_com_execute_admin_command(admin_queue,
1265                                             (struct ena_admin_aq_entry *)&destroy_cmd,
1266                                             sizeof(destroy_cmd),
1267                                             (struct ena_admin_acq_entry *)&destroy_resp,
1268                                             sizeof(destroy_resp));
1269
1270         if (unlikely(ret && (ret != -ENODEV)))
1271                 pr_err("Failed to destroy IO CQ. error: %d\n", ret);
1272
1273         return ret;
1274 }
1275
1276 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
1277 {
1278         return ena_dev->admin_queue.running_state;
1279 }
1280
1281 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
1282 {
1283         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1284         unsigned long flags;
1285
1286         spin_lock_irqsave(&admin_queue->q_lock, flags);
1287         ena_dev->admin_queue.running_state = state;
1288         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1289 }
1290
1291 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
1292 {
1293         u16 depth = ena_dev->aenq.q_depth;
1294
1295         WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
1296
1297         /* Init head_db to mark that all entries in the queue
1298          * are initially available
1299          */
1300         writel(depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1301 }
1302
1303 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
1304 {
1305         struct ena_com_admin_queue *admin_queue;
1306         struct ena_admin_set_feat_cmd cmd;
1307         struct ena_admin_set_feat_resp resp;
1308         struct ena_admin_get_feat_resp get_resp;
1309         int ret;
1310
1311         ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
1312         if (ret) {
1313                 pr_info("Can't get aenq configuration\n");
1314                 return ret;
1315         }
1316
1317         if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
1318                 pr_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
1319                         get_resp.u.aenq.supported_groups, groups_flag);
1320                 return -EPERM;
1321         }
1322
1323         memset(&cmd, 0x0, sizeof(cmd));
1324         admin_queue = &ena_dev->admin_queue;
1325
1326         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1327         cmd.aq_common_descriptor.flags = 0;
1328         cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
1329         cmd.u.aenq.enabled_groups = groups_flag;
1330
1331         ret = ena_com_execute_admin_command(admin_queue,
1332                                             (struct ena_admin_aq_entry *)&cmd,
1333                                             sizeof(cmd),
1334                                             (struct ena_admin_acq_entry *)&resp,
1335                                             sizeof(resp));
1336
1337         if (unlikely(ret))
1338                 pr_err("Failed to config AENQ ret: %d\n", ret);
1339
1340         return ret;
1341 }
1342
1343 int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
1344 {
1345         u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1346         int width;
1347
1348         if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
1349                 pr_err("Reg read timeout occurred\n");
1350                 return -ETIME;
1351         }
1352
1353         width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
1354                 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
1355
1356         pr_debug("ENA dma width: %d\n", width);
1357
1358         if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
1359                 pr_err("DMA width illegal value: %d\n", width);
1360                 return -EINVAL;
1361         }
1362
1363         ena_dev->dma_addr_bits = width;
1364
1365         return width;
1366 }
1367
1368 int ena_com_validate_version(struct ena_com_dev *ena_dev)
1369 {
1370         u32 ver;
1371         u32 ctrl_ver;
1372         u32 ctrl_ver_masked;
1373
1374         /* Make sure the ENA version and the controller version are at least
1375          * as the driver expects
1376          */
1377         ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
1378         ctrl_ver = ena_com_reg_bar_read32(ena_dev,
1379                                           ENA_REGS_CONTROLLER_VERSION_OFF);
1380
1381         if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
1382                      (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
1383                 pr_err("Reg read timeout occurred\n");
1384                 return -ETIME;
1385         }
1386
1387         pr_info("ena device version: %d.%d\n",
1388                 (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
1389                         ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
1390                 ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
1391
1392         if (ver < MIN_ENA_VER) {
1393                 pr_err("ENA version is lower than the minimal version the driver supports\n");
1394                 return -1;
1395         }
1396
1397         pr_info("ena controller version: %d.%d.%d implementation version %d\n",
1398                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
1399                         ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
1400                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
1401                         ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
1402                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
1403                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
1404                         ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
1405
1406         ctrl_ver_masked =
1407                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
1408                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
1409                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
1410
1411         /* Validate the ctrl version without the implementation ID */
1412         if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
1413                 pr_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
1414                 return -1;
1415         }
1416
1417         return 0;
1418 }
1419
1420 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
1421 {
1422         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1423         struct ena_com_admin_cq *cq = &admin_queue->cq;
1424         struct ena_com_admin_sq *sq = &admin_queue->sq;
1425         struct ena_com_aenq *aenq = &ena_dev->aenq;
1426         u16 size;
1427
1428         if (admin_queue->comp_ctx)
1429                 devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);
1430         admin_queue->comp_ctx = NULL;
1431         size = ADMIN_SQ_SIZE(admin_queue->q_depth);
1432         if (sq->entries)
1433                 dma_free_coherent(ena_dev->dmadev, size, sq->entries,
1434                                   sq->dma_addr);
1435         sq->entries = NULL;
1436
1437         size = ADMIN_CQ_SIZE(admin_queue->q_depth);
1438         if (cq->entries)
1439                 dma_free_coherent(ena_dev->dmadev, size, cq->entries,
1440                                   cq->dma_addr);
1441         cq->entries = NULL;
1442
1443         size = ADMIN_AENQ_SIZE(aenq->q_depth);
1444         if (ena_dev->aenq.entries)
1445                 dma_free_coherent(ena_dev->dmadev, size, aenq->entries,
1446                                   aenq->dma_addr);
1447         aenq->entries = NULL;
1448 }
1449
1450 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
1451 {
1452         ena_dev->admin_queue.polling = polling;
1453 }
1454
1455 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
1456 {
1457         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1458
1459         spin_lock_init(&mmio_read->lock);
1460         mmio_read->read_resp =
1461                 dma_zalloc_coherent(ena_dev->dmadev,
1462                                     sizeof(*mmio_read->read_resp),
1463                                     &mmio_read->read_resp_dma_addr, GFP_KERNEL);
1464         if (unlikely(!mmio_read->read_resp))
1465                 return -ENOMEM;
1466
1467         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1468
1469         mmio_read->read_resp->req_id = 0x0;
1470         mmio_read->seq_num = 0x0;
1471         mmio_read->readless_supported = true;
1472
1473         return 0;
1474 }
1475
1476 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
1477 {
1478         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1479
1480         mmio_read->readless_supported = readless_supported;
1481 }
1482
1483 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
1484 {
1485         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1486
1487         writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1488         writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1489
1490         dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp),
1491                           mmio_read->read_resp, mmio_read->read_resp_dma_addr);
1492
1493         mmio_read->read_resp = NULL;
1494 }
1495
1496 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
1497 {
1498         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1499         u32 addr_low, addr_high;
1500
1501         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
1502         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
1503
1504         writel(addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1505         writel(addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1506 }
1507
1508 int ena_com_admin_init(struct ena_com_dev *ena_dev,
1509                        struct ena_aenq_handlers *aenq_handlers,
1510                        bool init_spinlock)
1511 {
1512         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1513         u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
1514         int ret;
1515
1516         dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1517
1518         if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
1519                 pr_err("Reg read timeout occurred\n");
1520                 return -ETIME;
1521         }
1522
1523         if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
1524                 pr_err("Device isn't ready, abort com init\n");
1525                 return -ENODEV;
1526         }
1527
1528         admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
1529
1530         admin_queue->q_dmadev = ena_dev->dmadev;
1531         admin_queue->polling = false;
1532         admin_queue->curr_cmd_id = 0;
1533
1534         atomic_set(&admin_queue->outstanding_cmds, 0);
1535
1536         if (init_spinlock)
1537                 spin_lock_init(&admin_queue->q_lock);
1538
1539         ret = ena_com_init_comp_ctxt(admin_queue);
1540         if (ret)
1541                 goto error;
1542
1543         ret = ena_com_admin_init_sq(admin_queue);
1544         if (ret)
1545                 goto error;
1546
1547         ret = ena_com_admin_init_cq(admin_queue);
1548         if (ret)
1549                 goto error;
1550
1551         admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1552                 ENA_REGS_AQ_DB_OFF);
1553
1554         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
1555         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
1556
1557         writel(addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
1558         writel(addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
1559
1560         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
1561         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
1562
1563         writel(addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
1564         writel(addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
1565
1566         aq_caps = 0;
1567         aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
1568         aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
1569                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
1570                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
1571
1572         acq_caps = 0;
1573         acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
1574         acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
1575                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
1576                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
1577
1578         writel(aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
1579         writel(acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
1580         ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
1581         if (ret)
1582                 goto error;
1583
1584         admin_queue->running_state = true;
1585
1586         return 0;
1587 error:
1588         ena_com_admin_destroy(ena_dev);
1589
1590         return ret;
1591 }
1592
1593 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
1594                             struct ena_com_create_io_ctx *ctx)
1595 {
1596         struct ena_com_io_sq *io_sq;
1597         struct ena_com_io_cq *io_cq;
1598         int ret;
1599
1600         if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
1601                 pr_err("Qid (%d) is bigger than max num of queues (%d)\n",
1602                        ctx->qid, ENA_TOTAL_NUM_QUEUES);
1603                 return -EINVAL;
1604         }
1605
1606         io_sq = &ena_dev->io_sq_queues[ctx->qid];
1607         io_cq = &ena_dev->io_cq_queues[ctx->qid];
1608
1609         memset(io_sq, 0x0, sizeof(struct ena_com_io_sq));
1610         memset(io_cq, 0x0, sizeof(struct ena_com_io_cq));
1611
1612         /* Init CQ */
1613         io_cq->q_depth = ctx->queue_size;
1614         io_cq->direction = ctx->direction;
1615         io_cq->qid = ctx->qid;
1616
1617         io_cq->msix_vector = ctx->msix_vector;
1618
1619         io_sq->q_depth = ctx->queue_size;
1620         io_sq->direction = ctx->direction;
1621         io_sq->qid = ctx->qid;
1622
1623         io_sq->mem_queue_type = ctx->mem_queue_type;
1624
1625         if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1626                 /* header length is limited to 8 bits */
1627                 io_sq->tx_max_header_size =
1628                         min_t(u32, ena_dev->tx_max_header_size, SZ_256);
1629
1630         ret = ena_com_init_io_sq(ena_dev, ctx, io_sq);
1631         if (ret)
1632                 goto error;
1633         ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
1634         if (ret)
1635                 goto error;
1636
1637         ret = ena_com_create_io_cq(ena_dev, io_cq);
1638         if (ret)
1639                 goto error;
1640
1641         ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
1642         if (ret)
1643                 goto destroy_io_cq;
1644
1645         return 0;
1646
1647 destroy_io_cq:
1648         ena_com_destroy_io_cq(ena_dev, io_cq);
1649 error:
1650         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1651         return ret;
1652 }
1653
1654 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
1655 {
1656         struct ena_com_io_sq *io_sq;
1657         struct ena_com_io_cq *io_cq;
1658
1659         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1660                 pr_err("Qid (%d) is bigger than max num of queues (%d)\n", qid,
1661                        ENA_TOTAL_NUM_QUEUES);
1662                 return;
1663         }
1664
1665         io_sq = &ena_dev->io_sq_queues[qid];
1666         io_cq = &ena_dev->io_cq_queues[qid];
1667
1668         ena_com_destroy_io_sq(ena_dev, io_sq);
1669         ena_com_destroy_io_cq(ena_dev, io_cq);
1670
1671         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1672 }
1673
1674 int ena_com_get_link_params(struct ena_com_dev *ena_dev,
1675                             struct ena_admin_get_feat_resp *resp)
1676 {
1677         return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG);
1678 }
1679
1680 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
1681                               struct ena_com_dev_get_features_ctx *get_feat_ctx)
1682 {
1683         struct ena_admin_get_feat_resp get_resp;
1684         int rc;
1685
1686         rc = ena_com_get_feature(ena_dev, &get_resp,
1687                                  ENA_ADMIN_DEVICE_ATTRIBUTES);
1688         if (rc)
1689                 return rc;
1690
1691         memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
1692                sizeof(get_resp.u.dev_attr));
1693         ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
1694
1695         rc = ena_com_get_feature(ena_dev, &get_resp,
1696                                  ENA_ADMIN_MAX_QUEUES_NUM);
1697         if (rc)
1698                 return rc;
1699
1700         memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
1701                sizeof(get_resp.u.max_queue));
1702         ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size;
1703
1704         rc = ena_com_get_feature(ena_dev, &get_resp,
1705                                  ENA_ADMIN_AENQ_CONFIG);
1706         if (rc)
1707                 return rc;
1708
1709         memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
1710                sizeof(get_resp.u.aenq));
1711
1712         rc = ena_com_get_feature(ena_dev, &get_resp,
1713                                  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1714         if (rc)
1715                 return rc;
1716
1717         memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
1718                sizeof(get_resp.u.offload));
1719
1720         return 0;
1721 }
1722
1723 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
1724 {
1725         ena_com_handle_admin_completion(&ena_dev->admin_queue);
1726 }
1727
1728 /* ena_handle_specific_aenq_event:
1729  * return the handler that is relevant to the specific event group
1730  */
1731 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
1732                                                      u16 group)
1733 {
1734         struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
1735
1736         if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
1737                 return aenq_handlers->handlers[group];
1738
1739         return aenq_handlers->unimplemented_handler;
1740 }
1741
1742 /* ena_aenq_intr_handler:
1743  * handles the aenq incoming events.
1744  * pop events from the queue and apply the specific handler
1745  */
1746 void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
1747 {
1748         struct ena_admin_aenq_entry *aenq_e;
1749         struct ena_admin_aenq_common_desc *aenq_common;
1750         struct ena_com_aenq *aenq  = &dev->aenq;
1751         ena_aenq_handler handler_cb;
1752         u16 masked_head, processed = 0;
1753         u8 phase;
1754
1755         masked_head = aenq->head & (aenq->q_depth - 1);
1756         phase = aenq->phase;
1757         aenq_e = &aenq->entries[masked_head]; /* Get first entry */
1758         aenq_common = &aenq_e->aenq_common_desc;
1759
1760         /* Go over all the events */
1761         while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
1762                phase) {
1763                 pr_debug("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
1764                          aenq_common->group, aenq_common->syndrom,
1765                          (u64)aenq_common->timestamp_low +
1766                                  ((u64)aenq_common->timestamp_high << 32));
1767
1768                 /* Handle specific event*/
1769                 handler_cb = ena_com_get_specific_aenq_cb(dev,
1770                                                           aenq_common->group);
1771                 handler_cb(data, aenq_e); /* call the actual event handler*/
1772
1773                 /* Get next event entry */
1774                 masked_head++;
1775                 processed++;
1776
1777                 if (unlikely(masked_head == aenq->q_depth)) {
1778                         masked_head = 0;
1779                         phase = !phase;
1780                 }
1781                 aenq_e = &aenq->entries[masked_head];
1782                 aenq_common = &aenq_e->aenq_common_desc;
1783         }
1784
1785         aenq->head += processed;
1786         aenq->phase = phase;
1787
1788         /* Don't update aenq doorbell if there weren't any processed events */
1789         if (!processed)
1790                 return;
1791
1792         /* write the aenq doorbell after all AENQ descriptors were read */
1793         mb();
1794         writel((u32)aenq->head, dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1795 }
1796
1797 int ena_com_dev_reset(struct ena_com_dev *ena_dev)
1798 {
1799         u32 stat, timeout, cap, reset_val;
1800         int rc;
1801
1802         stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1803         cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1804
1805         if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
1806                      (cap == ENA_MMIO_READ_TIMEOUT))) {
1807                 pr_err("Reg read32 timeout occurred\n");
1808                 return -ETIME;
1809         }
1810
1811         if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
1812                 pr_err("Device isn't ready, can't reset device\n");
1813                 return -EINVAL;
1814         }
1815
1816         timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
1817                         ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
1818         if (timeout == 0) {
1819                 pr_err("Invalid timeout value\n");
1820                 return -EINVAL;
1821         }
1822
1823         /* start reset */
1824         reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
1825         writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1826
1827         /* Write again the MMIO read request address */
1828         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1829
1830         rc = wait_for_reset_state(ena_dev, timeout,
1831                                   ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
1832         if (rc != 0) {
1833                 pr_err("Reset indication didn't turn on\n");
1834                 return rc;
1835         }
1836
1837         /* reset done */
1838         writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1839         rc = wait_for_reset_state(ena_dev, timeout, 0);
1840         if (rc != 0) {
1841                 pr_err("Reset indication didn't turn off\n");
1842                 return rc;
1843         }
1844
1845         return 0;
1846 }
1847
1848 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
1849                              struct ena_com_stats_ctx *ctx,
1850                              enum ena_admin_get_stats_type type)
1851 {
1852         struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
1853         struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
1854         struct ena_com_admin_queue *admin_queue;
1855         int ret;
1856
1857         admin_queue = &ena_dev->admin_queue;
1858
1859         get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
1860         get_cmd->aq_common_descriptor.flags = 0;
1861         get_cmd->type = type;
1862
1863         ret =  ena_com_execute_admin_command(admin_queue,
1864                                              (struct ena_admin_aq_entry *)get_cmd,
1865                                              sizeof(*get_cmd),
1866                                              (struct ena_admin_acq_entry *)get_resp,
1867                                              sizeof(*get_resp));
1868
1869         if (unlikely(ret))
1870                 pr_err("Failed to get stats. error: %d\n", ret);
1871
1872         return ret;
1873 }
1874
1875 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
1876                                 struct ena_admin_basic_stats *stats)
1877 {
1878         struct ena_com_stats_ctx ctx;
1879         int ret;
1880
1881         memset(&ctx, 0x0, sizeof(ctx));
1882         ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
1883         if (likely(ret == 0))
1884                 memcpy(stats, &ctx.get_resp.basic_stats,
1885                        sizeof(ctx.get_resp.basic_stats));
1886
1887         return ret;
1888 }
1889
1890 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
1891 {
1892         struct ena_com_admin_queue *admin_queue;
1893         struct ena_admin_set_feat_cmd cmd;
1894         struct ena_admin_set_feat_resp resp;
1895         int ret;
1896
1897         if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
1898                 pr_info("Feature %d isn't supported\n", ENA_ADMIN_MTU);
1899                 return -EPERM;
1900         }
1901
1902         memset(&cmd, 0x0, sizeof(cmd));
1903         admin_queue = &ena_dev->admin_queue;
1904
1905         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1906         cmd.aq_common_descriptor.flags = 0;
1907         cmd.feat_common.feature_id = ENA_ADMIN_MTU;
1908         cmd.u.mtu.mtu = mtu;
1909
1910         ret = ena_com_execute_admin_command(admin_queue,
1911                                             (struct ena_admin_aq_entry *)&cmd,
1912                                             sizeof(cmd),
1913                                             (struct ena_admin_acq_entry *)&resp,
1914                                             sizeof(resp));
1915
1916         if (unlikely(ret))
1917                 pr_err("Failed to set mtu %d. error: %d\n", mtu, ret);
1918
1919         return ret;
1920 }
1921
1922 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
1923                                  struct ena_admin_feature_offload_desc *offload)
1924 {
1925         int ret;
1926         struct ena_admin_get_feat_resp resp;
1927
1928         ret = ena_com_get_feature(ena_dev, &resp,
1929                                   ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1930         if (unlikely(ret)) {
1931                 pr_err("Failed to get offload capabilities %d\n", ret);
1932                 return ret;
1933         }
1934
1935         memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
1936
1937         return 0;
1938 }
1939
1940 int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
1941 {
1942         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1943         struct ena_rss *rss = &ena_dev->rss;
1944         struct ena_admin_set_feat_cmd cmd;
1945         struct ena_admin_set_feat_resp resp;
1946         struct ena_admin_get_feat_resp get_resp;
1947         int ret;
1948
1949         if (!ena_com_check_supported_feature_id(ena_dev,
1950                                                 ENA_ADMIN_RSS_HASH_FUNCTION)) {
1951                 pr_info("Feature %d isn't supported\n",
1952                         ENA_ADMIN_RSS_HASH_FUNCTION);
1953                 return -EPERM;
1954         }
1955
1956         /* Validate hash function is supported */
1957         ret = ena_com_get_feature(ena_dev, &get_resp,
1958                                   ENA_ADMIN_RSS_HASH_FUNCTION);
1959         if (unlikely(ret))
1960                 return ret;
1961
1962         if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
1963                 pr_err("Func hash %d isn't supported by device, abort\n",
1964                        rss->hash_func);
1965                 return -EPERM;
1966         }
1967
1968         memset(&cmd, 0x0, sizeof(cmd));
1969
1970         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1971         cmd.aq_common_descriptor.flags =
1972                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
1973         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
1974         cmd.u.flow_hash_func.init_val = rss->hash_init_val;
1975         cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
1976
1977         ret = ena_com_mem_addr_set(ena_dev,
1978                                    &cmd.control_buffer.address,
1979                                    rss->hash_key_dma_addr);
1980         if (unlikely(ret)) {
1981                 pr_err("memory address set failed\n");
1982                 return ret;
1983         }
1984
1985         cmd.control_buffer.length = sizeof(*rss->hash_key);
1986
1987         ret = ena_com_execute_admin_command(admin_queue,
1988                                             (struct ena_admin_aq_entry *)&cmd,
1989                                             sizeof(cmd),
1990                                             (struct ena_admin_acq_entry *)&resp,
1991                                             sizeof(resp));
1992         if (unlikely(ret)) {
1993                 pr_err("Failed to set hash function %d. error: %d\n",
1994                        rss->hash_func, ret);
1995                 return -EINVAL;
1996         }
1997
1998         return 0;
1999 }
2000
2001 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
2002                                enum ena_admin_hash_functions func,
2003                                const u8 *key, u16 key_len, u32 init_val)
2004 {
2005         struct ena_rss *rss = &ena_dev->rss;
2006         struct ena_admin_get_feat_resp get_resp;
2007         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2008                 rss->hash_key;
2009         int rc;
2010
2011         /* Make sure size is a mult of DWs */
2012         if (unlikely(key_len & 0x3))
2013                 return -EINVAL;
2014
2015         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2016                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2017                                     rss->hash_key_dma_addr,
2018                                     sizeof(*rss->hash_key));
2019         if (unlikely(rc))
2020                 return rc;
2021
2022         if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
2023                 pr_err("Flow hash function %d isn't supported\n", func);
2024                 return -EPERM;
2025         }
2026
2027         switch (func) {
2028         case ENA_ADMIN_TOEPLITZ:
2029                 if (key_len > sizeof(hash_key->key)) {
2030                         pr_err("key len (%hu) is bigger than the max supported (%zu)\n",
2031                                key_len, sizeof(hash_key->key));
2032                         return -EINVAL;
2033                 }
2034
2035                 memcpy(hash_key->key, key, key_len);
2036                 rss->hash_init_val = init_val;
2037                 hash_key->keys_num = key_len >> 2;
2038                 break;
2039         case ENA_ADMIN_CRC32:
2040                 rss->hash_init_val = init_val;
2041                 break;
2042         default:
2043                 pr_err("Invalid hash function (%d)\n", func);
2044                 return -EINVAL;
2045         }
2046
2047         rc = ena_com_set_hash_function(ena_dev);
2048
2049         /* Restore the old function */
2050         if (unlikely(rc))
2051                 ena_com_get_hash_function(ena_dev, NULL, NULL);
2052
2053         return rc;
2054 }
2055
2056 int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
2057                               enum ena_admin_hash_functions *func,
2058                               u8 *key)
2059 {
2060         struct ena_rss *rss = &ena_dev->rss;
2061         struct ena_admin_get_feat_resp get_resp;
2062         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2063                 rss->hash_key;
2064         int rc;
2065
2066         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2067                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2068                                     rss->hash_key_dma_addr,
2069                                     sizeof(*rss->hash_key));
2070         if (unlikely(rc))
2071                 return rc;
2072
2073         rss->hash_func = get_resp.u.flow_hash_func.selected_func;
2074         if (func)
2075                 *func = rss->hash_func;
2076
2077         if (key)
2078                 memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2);
2079
2080         return 0;
2081 }
2082
2083 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
2084                           enum ena_admin_flow_hash_proto proto,
2085                           u16 *fields)
2086 {
2087         struct ena_rss *rss = &ena_dev->rss;
2088         struct ena_admin_get_feat_resp get_resp;
2089         int rc;
2090
2091         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2092                                     ENA_ADMIN_RSS_HASH_INPUT,
2093                                     rss->hash_ctrl_dma_addr,
2094                                     sizeof(*rss->hash_ctrl));
2095         if (unlikely(rc))
2096                 return rc;
2097
2098         if (fields)
2099                 *fields = rss->hash_ctrl->selected_fields[proto].fields;
2100
2101         return 0;
2102 }
2103
2104 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
2105 {
2106         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2107         struct ena_rss *rss = &ena_dev->rss;
2108         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2109         struct ena_admin_set_feat_cmd cmd;
2110         struct ena_admin_set_feat_resp resp;
2111         int ret;
2112
2113         if (!ena_com_check_supported_feature_id(ena_dev,
2114                                                 ENA_ADMIN_RSS_HASH_INPUT)) {
2115                 pr_info("Feature %d isn't supported\n", ENA_ADMIN_RSS_HASH_INPUT);
2116                 return -EPERM;
2117         }
2118
2119         memset(&cmd, 0x0, sizeof(cmd));
2120
2121         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2122         cmd.aq_common_descriptor.flags =
2123                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2124         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
2125         cmd.u.flow_hash_input.enabled_input_sort =
2126                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
2127                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
2128
2129         ret = ena_com_mem_addr_set(ena_dev,
2130                                    &cmd.control_buffer.address,
2131                                    rss->hash_ctrl_dma_addr);
2132         if (unlikely(ret)) {
2133                 pr_err("memory address set failed\n");
2134                 return ret;
2135         }
2136         cmd.control_buffer.length = sizeof(*hash_ctrl);
2137
2138         ret = ena_com_execute_admin_command(admin_queue,
2139                                             (struct ena_admin_aq_entry *)&cmd,
2140                                             sizeof(cmd),
2141                                             (struct ena_admin_acq_entry *)&resp,
2142                                             sizeof(resp));
2143         if (unlikely(ret))
2144                 pr_err("Failed to set hash input. error: %d\n", ret);
2145
2146         return ret;
2147 }
2148
2149 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
2150 {
2151         struct ena_rss *rss = &ena_dev->rss;
2152         struct ena_admin_feature_rss_hash_control *hash_ctrl =
2153                 rss->hash_ctrl;
2154         u16 available_fields = 0;
2155         int rc, i;
2156
2157         /* Get the supported hash input */
2158         rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2159         if (unlikely(rc))
2160                 return rc;
2161
2162         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
2163                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2164                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2165
2166         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
2167                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2168                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2169
2170         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
2171                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2172                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2173
2174         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
2175                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2176                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2177
2178         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
2179                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2180
2181         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
2182                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2183
2184         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2185                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2186
2187         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2188                 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
2189
2190         for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
2191                 available_fields = hash_ctrl->selected_fields[i].fields &
2192                                 hash_ctrl->supported_fields[i].fields;
2193                 if (available_fields != hash_ctrl->selected_fields[i].fields) {
2194                         pr_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
2195                                i, hash_ctrl->supported_fields[i].fields,
2196                                hash_ctrl->selected_fields[i].fields);
2197                         return -EPERM;
2198                 }
2199         }
2200
2201         rc = ena_com_set_hash_ctrl(ena_dev);
2202
2203         /* In case of failure, restore the old hash ctrl */
2204         if (unlikely(rc))
2205                 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2206
2207         return rc;
2208 }
2209
2210 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
2211                            enum ena_admin_flow_hash_proto proto,
2212                            u16 hash_fields)
2213 {
2214         struct ena_rss *rss = &ena_dev->rss;
2215         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2216         u16 supported_fields;
2217         int rc;
2218
2219         if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
2220                 pr_err("Invalid proto num (%u)\n", proto);
2221                 return -EINVAL;
2222         }
2223
2224         /* Get the ctrl table */
2225         rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
2226         if (unlikely(rc))
2227                 return rc;
2228
2229         /* Make sure all the fields are supported */
2230         supported_fields = hash_ctrl->supported_fields[proto].fields;
2231         if ((hash_fields & supported_fields) != hash_fields) {
2232                 pr_err("proto %d doesn't support the required fields %x. supports only: %x\n",
2233                        proto, hash_fields, supported_fields);
2234         }
2235
2236         hash_ctrl->selected_fields[proto].fields = hash_fields;
2237
2238         rc = ena_com_set_hash_ctrl(ena_dev);
2239
2240         /* In case of failure, restore the old hash ctrl */
2241         if (unlikely(rc))
2242                 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2243
2244         return 0;
2245 }
2246
2247 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
2248                                       u16 entry_idx, u16 entry_value)
2249 {
2250         struct ena_rss *rss = &ena_dev->rss;
2251
2252         if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
2253                 return -EINVAL;
2254
2255         if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
2256                 return -EINVAL;
2257
2258         rss->host_rss_ind_tbl[entry_idx] = entry_value;
2259
2260         return 0;
2261 }
2262
2263 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
2264 {
2265         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2266         struct ena_rss *rss = &ena_dev->rss;
2267         struct ena_admin_set_feat_cmd cmd;
2268         struct ena_admin_set_feat_resp resp;
2269         int ret;
2270
2271         if (!ena_com_check_supported_feature_id(
2272                     ena_dev, ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
2273                 pr_info("Feature %d isn't supported\n",
2274                         ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
2275                 return -EPERM;
2276         }
2277
2278         ret = ena_com_ind_tbl_convert_to_device(ena_dev);
2279         if (ret) {
2280                 pr_err("Failed to convert host indirection table to device table\n");
2281                 return ret;
2282         }
2283
2284         memset(&cmd, 0x0, sizeof(cmd));
2285
2286         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2287         cmd.aq_common_descriptor.flags =
2288                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2289         cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
2290         cmd.u.ind_table.size = rss->tbl_log_size;
2291         cmd.u.ind_table.inline_index = 0xFFFFFFFF;
2292
2293         ret = ena_com_mem_addr_set(ena_dev,
2294                                    &cmd.control_buffer.address,
2295                                    rss->rss_ind_tbl_dma_addr);
2296         if (unlikely(ret)) {
2297                 pr_err("memory address set failed\n");
2298                 return ret;
2299         }
2300
2301         cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
2302                 sizeof(struct ena_admin_rss_ind_table_entry);
2303
2304         ret = ena_com_execute_admin_command(admin_queue,
2305                                             (struct ena_admin_aq_entry *)&cmd,
2306                                             sizeof(cmd),
2307                                             (struct ena_admin_acq_entry *)&resp,
2308                                             sizeof(resp));
2309
2310         if (unlikely(ret))
2311                 pr_err("Failed to set indirect table. error: %d\n", ret);
2312
2313         return ret;
2314 }
2315
2316 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
2317 {
2318         struct ena_rss *rss = &ena_dev->rss;
2319         struct ena_admin_get_feat_resp get_resp;
2320         u32 tbl_size;
2321         int i, rc;
2322
2323         tbl_size = (1ULL << rss->tbl_log_size) *
2324                 sizeof(struct ena_admin_rss_ind_table_entry);
2325
2326         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2327                                     ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
2328                                     rss->rss_ind_tbl_dma_addr,
2329                                     tbl_size);
2330         if (unlikely(rc))
2331                 return rc;
2332
2333         if (!ind_tbl)
2334                 return 0;
2335
2336         rc = ena_com_ind_tbl_convert_from_device(ena_dev);
2337         if (unlikely(rc))
2338                 return rc;
2339
2340         for (i = 0; i < (1 << rss->tbl_log_size); i++)
2341                 ind_tbl[i] = rss->host_rss_ind_tbl[i];
2342
2343         return 0;
2344 }
2345
2346 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
2347 {
2348         int rc;
2349
2350         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2351
2352         rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
2353         if (unlikely(rc))
2354                 goto err_indr_tbl;
2355
2356         rc = ena_com_hash_key_allocate(ena_dev);
2357         if (unlikely(rc))
2358                 goto err_hash_key;
2359
2360         rc = ena_com_hash_ctrl_init(ena_dev);
2361         if (unlikely(rc))
2362                 goto err_hash_ctrl;
2363
2364         return 0;
2365
2366 err_hash_ctrl:
2367         ena_com_hash_key_destroy(ena_dev);
2368 err_hash_key:
2369         ena_com_indirect_table_destroy(ena_dev);
2370 err_indr_tbl:
2371
2372         return rc;
2373 }
2374
2375 void ena_com_rss_destroy(struct ena_com_dev *ena_dev)
2376 {
2377         ena_com_indirect_table_destroy(ena_dev);
2378         ena_com_hash_key_destroy(ena_dev);
2379         ena_com_hash_ctrl_destroy(ena_dev);
2380
2381         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2382 }
2383
2384 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
2385 {
2386         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2387
2388         host_attr->host_info =
2389                 dma_zalloc_coherent(ena_dev->dmadev, SZ_4K,
2390                                     &host_attr->host_info_dma_addr, GFP_KERNEL);
2391         if (unlikely(!host_attr->host_info))
2392                 return -ENOMEM;
2393
2394         return 0;
2395 }
2396
2397 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
2398                                 u32 debug_area_size)
2399 {
2400         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2401
2402         host_attr->debug_area_virt_addr =
2403                 dma_zalloc_coherent(ena_dev->dmadev, debug_area_size,
2404                                     &host_attr->debug_area_dma_addr, GFP_KERNEL);
2405         if (unlikely(!host_attr->debug_area_virt_addr)) {
2406                 host_attr->debug_area_size = 0;
2407                 return -ENOMEM;
2408         }
2409
2410         host_attr->debug_area_size = debug_area_size;
2411
2412         return 0;
2413 }
2414
2415 void ena_com_delete_host_info(struct ena_com_dev *ena_dev)
2416 {
2417         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2418
2419         if (host_attr->host_info) {
2420                 dma_free_coherent(ena_dev->dmadev, SZ_4K, host_attr->host_info,
2421                                   host_attr->host_info_dma_addr);
2422                 host_attr->host_info = NULL;
2423         }
2424 }
2425
2426 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev)
2427 {
2428         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2429
2430         if (host_attr->debug_area_virt_addr) {
2431                 dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size,
2432                                   host_attr->debug_area_virt_addr,
2433                                   host_attr->debug_area_dma_addr);
2434                 host_attr->debug_area_virt_addr = NULL;
2435         }
2436 }
2437
2438 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
2439 {
2440         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2441         struct ena_com_admin_queue *admin_queue;
2442         struct ena_admin_set_feat_cmd cmd;
2443         struct ena_admin_set_feat_resp resp;
2444
2445         int ret;
2446
2447         if (!ena_com_check_supported_feature_id(ena_dev,
2448                                                 ENA_ADMIN_HOST_ATTR_CONFIG)) {
2449                 pr_warn("Set host attribute isn't supported\n");
2450                 return -EPERM;
2451         }
2452
2453         memset(&cmd, 0x0, sizeof(cmd));
2454         admin_queue = &ena_dev->admin_queue;
2455
2456         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2457         cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
2458
2459         ret = ena_com_mem_addr_set(ena_dev,
2460                                    &cmd.u.host_attr.debug_ba,
2461                                    host_attr->debug_area_dma_addr);
2462         if (unlikely(ret)) {
2463                 pr_err("memory address set failed\n");
2464                 return ret;
2465         }
2466
2467         ret = ena_com_mem_addr_set(ena_dev,
2468                                    &cmd.u.host_attr.os_info_ba,
2469                                    host_attr->host_info_dma_addr);
2470         if (unlikely(ret)) {
2471                 pr_err("memory address set failed\n");
2472                 return ret;
2473         }
2474
2475         cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
2476
2477         ret = ena_com_execute_admin_command(admin_queue,
2478                                             (struct ena_admin_aq_entry *)&cmd,
2479                                             sizeof(cmd),
2480                                             (struct ena_admin_acq_entry *)&resp,
2481                                             sizeof(resp));
2482
2483         if (unlikely(ret))
2484                 pr_err("Failed to set host attributes: %d\n", ret);
2485
2486         return ret;
2487 }
2488
2489 /* Interrupt moderation */
2490 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
2491 {
2492         return ena_com_check_supported_feature_id(ena_dev,
2493                                                   ENA_ADMIN_INTERRUPT_MODERATION);
2494 }
2495
2496 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
2497                                                       u32 tx_coalesce_usecs)
2498 {
2499         if (!ena_dev->intr_delay_resolution) {
2500                 pr_err("Illegal interrupt delay granularity value\n");
2501                 return -EFAULT;
2502         }
2503
2504         ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
2505                 ena_dev->intr_delay_resolution;
2506
2507         return 0;
2508 }
2509
2510 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
2511                                                       u32 rx_coalesce_usecs)
2512 {
2513         if (!ena_dev->intr_delay_resolution) {
2514                 pr_err("Illegal interrupt delay granularity value\n");
2515                 return -EFAULT;
2516         }
2517
2518         /* We use LOWEST entry of moderation table for storing
2519          * nonadaptive interrupt coalescing values
2520          */
2521         ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2522                 rx_coalesce_usecs / ena_dev->intr_delay_resolution;
2523
2524         return 0;
2525 }
2526
2527 void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
2528 {
2529         if (ena_dev->intr_moder_tbl)
2530                 devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl);
2531         ena_dev->intr_moder_tbl = NULL;
2532 }
2533
2534 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
2535 {
2536         struct ena_admin_get_feat_resp get_resp;
2537         u16 delay_resolution;
2538         int rc;
2539
2540         rc = ena_com_get_feature(ena_dev, &get_resp,
2541                                  ENA_ADMIN_INTERRUPT_MODERATION);
2542
2543         if (rc) {
2544                 if (rc == -EPERM) {
2545                         pr_info("Feature %d isn't supported\n",
2546                                 ENA_ADMIN_INTERRUPT_MODERATION);
2547                         rc = 0;
2548                 } else {
2549                         pr_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
2550                                rc);
2551                 }
2552
2553                 /* no moderation supported, disable adaptive support */
2554                 ena_com_disable_adaptive_moderation(ena_dev);
2555                 return rc;
2556         }
2557
2558         rc = ena_com_init_interrupt_moderation_table(ena_dev);
2559         if (rc)
2560                 goto err;
2561
2562         /* if moderation is supported by device we set adaptive moderation */
2563         delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
2564         ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
2565         ena_com_enable_adaptive_moderation(ena_dev);
2566
2567         return 0;
2568 err:
2569         ena_com_destroy_interrupt_moderation(ena_dev);
2570         return rc;
2571 }
2572
2573 void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
2574 {
2575         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2576
2577         if (!intr_moder_tbl)
2578                 return;
2579
2580         intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2581                 ENA_INTR_LOWEST_USECS;
2582         intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
2583                 ENA_INTR_LOWEST_PKTS;
2584         intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
2585                 ENA_INTR_LOWEST_BYTES;
2586
2587         intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
2588                 ENA_INTR_LOW_USECS;
2589         intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
2590                 ENA_INTR_LOW_PKTS;
2591         intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
2592                 ENA_INTR_LOW_BYTES;
2593
2594         intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
2595                 ENA_INTR_MID_USECS;
2596         intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
2597                 ENA_INTR_MID_PKTS;
2598         intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
2599                 ENA_INTR_MID_BYTES;
2600
2601         intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
2602                 ENA_INTR_HIGH_USECS;
2603         intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
2604                 ENA_INTR_HIGH_PKTS;
2605         intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
2606                 ENA_INTR_HIGH_BYTES;
2607
2608         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
2609                 ENA_INTR_HIGHEST_USECS;
2610         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
2611                 ENA_INTR_HIGHEST_PKTS;
2612         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
2613                 ENA_INTR_HIGHEST_BYTES;
2614 }
2615
2616 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
2617 {
2618         return ena_dev->intr_moder_tx_interval;
2619 }
2620
2621 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
2622 {
2623         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2624
2625         if (intr_moder_tbl)
2626                 return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
2627
2628         return 0;
2629 }
2630
2631 void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
2632                                         enum ena_intr_moder_level level,
2633                                         struct ena_intr_moder_entry *entry)
2634 {
2635         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2636
2637         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2638                 return;
2639
2640         intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
2641         if (ena_dev->intr_delay_resolution)
2642                 intr_moder_tbl[level].intr_moder_interval /=
2643                         ena_dev->intr_delay_resolution;
2644         intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
2645
2646         /* use hardcoded value until ethtool supports bytecount parameter */
2647         if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
2648                 intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
2649 }
2650
2651 void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
2652                                        enum ena_intr_moder_level level,
2653                                        struct ena_intr_moder_entry *entry)
2654 {
2655         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2656
2657         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2658                 return;
2659
2660         entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
2661         if (ena_dev->intr_delay_resolution)
2662                 entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
2663         entry->pkts_per_interval =
2664         intr_moder_tbl[level].pkts_per_interval;
2665         entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
2666 }