drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c

   1 /*
   2  * Copyright (c) 2013, Mellanox Technologies inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <asm-generic/kmap_types.h>
  34 #include <linux/kernel.h>
  35 #include <linux/module.h>
  36 #include <linux/mlx5/driver.h>
  37 #include <linux/mlx5/cmd.h>
  38 #include "mlx5_core.h"
  39
  40 enum {
  41         MLX5_PAGES_CANT_GIVE    = 0,
  42         MLX5_PAGES_GIVE         = 1,
  43         MLX5_PAGES_TAKE         = 2
  44 };
  45
  46 enum {
  47         MLX5_BOOT_PAGES         = 1,
  48         MLX5_INIT_PAGES         = 2,
  49         MLX5_POST_INIT_PAGES    = 3
  50 };
  51
  52 struct mlx5_pages_req {
  53         struct mlx5_core_dev *dev;
  54         u32     func_id;
  55         s32     npages;
  56         struct work_struct work;
  57 };
  58
  59 struct fw_page {
  60         struct rb_node  rb_node;
  61         u64             addr;
  62         struct page     *page;
  63         u16             func_id;
  64 };
  65
  66 struct mlx5_query_pages_inbox {
  67         struct mlx5_inbox_hdr   hdr;
  68         u8                      rsvd[8];
  69 };
  70
  71 struct mlx5_query_pages_outbox {
  72         struct mlx5_outbox_hdr  hdr;
  73         __be16                  rsvd;
  74         __be16                  func_id;
  75         __be32                  num_pages;
  76 };
  77
  78 struct mlx5_manage_pages_inbox {
  79         struct mlx5_inbox_hdr   hdr;
  80         __be16                  rsvd;
  81         __be16                  func_id;
  82         __be32                  num_entries;
  83         __be64                  pas[0];
  84 };
  85
  86 struct mlx5_manage_pages_outbox {
  87         struct mlx5_outbox_hdr  hdr;
  88         __be32                  num_entries;
  89         u8                      rsvd[4];
  90         __be64                  pas[0];
  91 };
  92
  93 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)
  94 {
  95         struct rb_root *root = &dev->priv.page_root;
  96         struct rb_node **new = &root->rb_node;
  97         struct rb_node *parent = NULL;
  98         struct fw_page *nfp;
  99         struct fw_page *tfp;
 100
 101         while (*new) {
 102                 parent = *new;
 103                 tfp = rb_entry(parent, struct fw_page, rb_node);
 104                 if (tfp->addr < addr)
 105                         new = &parent->rb_left;
 106                 else if (tfp->addr > addr)
 107                         new = &parent->rb_right;
 108                 else
 109                         return -EEXIST;
 110         }
 111
 112         nfp = kmalloc(sizeof(*nfp), GFP_KERNEL);
 113         if (!nfp)
 114                 return -ENOMEM;
 115
 116         nfp->addr = addr;
 117         nfp->page = page;
 118         nfp->func_id = func_id;
 119
 120         rb_link_node(&nfp->rb_node, parent, new);
 121         rb_insert_color(&nfp->rb_node, root);
 122
 123         return 0;
 124 }
 125
 126 static struct page *remove_page(struct mlx5_core_dev *dev, u64 addr)
 127 {
 128         struct rb_root *root = &dev->priv.page_root;
 129         struct rb_node *tmp = root->rb_node;
 130         struct page *result = NULL;
 131         struct fw_page *tfp;
 132
 133         while (tmp) {
 134                 tfp = rb_entry(tmp, struct fw_page, rb_node);
 135                 if (tfp->addr < addr) {
 136                         tmp = tmp->rb_left;
 137                 } else if (tfp->addr > addr) {
 138                         tmp = tmp->rb_right;
 139                 } else {
 140                         rb_erase(&tfp->rb_node, root);
 141                         result = tfp->page;
 142                         kfree(tfp);
 143                         break;
 144                 }
 145         }
 146
 147         return result;
 148 }
 149
 150 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
 151                                 s32 *npages, int boot)
 152 {
 153         struct mlx5_query_pages_inbox   in;
 154         struct mlx5_query_pages_outbox  out;
 155         int err;
 156
 157         memset(&in, 0, sizeof(in));
 158         memset(&out, 0, sizeof(out));
 159         in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_PAGES);
 160         in.hdr.opmod = boot ? cpu_to_be16(MLX5_BOOT_PAGES) : cpu_to_be16(MLX5_INIT_PAGES);
 161
 162         err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
 163         if (err)
 164                 return err;
 165
 166         if (out.hdr.status)
 167                 return mlx5_cmd_status_to_err(&out.hdr);
 168
 169         *npages = be32_to_cpu(out.num_pages);
 170         *func_id = be16_to_cpu(out.func_id);
 171
 172         return err;
 173 }
 174
 175 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 176                       int notify_fail)
 177 {
 178         struct mlx5_manage_pages_inbox *in;
 179         struct mlx5_manage_pages_outbox out;
 180         struct page *page;
 181         int inlen;
 182         u64 addr;
 183         int err;
 184         int i;
 185
 186         inlen = sizeof(*in) + npages * sizeof(in->pas[0]);
 187         in = mlx5_vzalloc(inlen);
 188         if (!in) {
 189                 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
 190                 return -ENOMEM;
 191         }
 192         memset(&out, 0, sizeof(out));
 193
 194         for (i = 0; i < npages; i++) {
 195                 page = alloc_page(GFP_HIGHUSER);
 196                 if (!page) {
 197                         err = -ENOMEM;
 198                         mlx5_core_warn(dev, "failed to allocate page\n");
 199                         goto out_alloc;
 200                 }
 201                 addr = dma_map_page(&dev->pdev->dev, page, 0,
 202                                     PAGE_SIZE, DMA_BIDIRECTIONAL);
 203                 if (dma_mapping_error(&dev->pdev->dev, addr)) {
 204                         mlx5_core_warn(dev, "failed dma mapping page\n");
 205                         __free_page(page);
 206                         err = -ENOMEM;
 207                         goto out_alloc;
 208                 }
 209                 err = insert_page(dev, addr, page, func_id);
 210                 if (err) {
 211                         mlx5_core_err(dev, "failed to track allocated page\n");
 212                         dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 213                         __free_page(page);
 214                         err = -ENOMEM;
 215                         goto out_alloc;
 216                 }
 217                 in->pas[i] = cpu_to_be64(addr);
 218         }
 219
 220         in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 221         in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE);
 222         in->func_id = cpu_to_be16(func_id);
 223         in->num_entries = cpu_to_be32(npages);
 224         err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
 225         mlx5_core_dbg(dev, "err %d\n", err);
 226         if (err) {
 227                 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", func_id, npages, err);
 228                 goto out_alloc;
 229         }
 230         dev->priv.fw_pages += npages;
 231
 232         if (out.hdr.status) {
 233                 err = mlx5_cmd_status_to_err(&out.hdr);
 234                 if (err) {
 235                         mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", func_id, npages, out.hdr.status);
 236                         goto out_alloc;
 237                 }
 238         }
 239
 240         mlx5_core_dbg(dev, "err %d\n", err);
 241
 242         goto out_free;
 243
 244 out_alloc:
 245         if (notify_fail) {
 246                 memset(in, 0, inlen);
 247                 memset(&out, 0, sizeof(out));
 248                 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 249                 in->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
 250                 if (mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out)))
 251                         mlx5_core_warn(dev, "\n");
 252         }
 253         for (i--; i >= 0; i--) {
 254                 addr = be64_to_cpu(in->pas[i]);
 255                 page = remove_page(dev, addr);
 256                 if (!page) {
 257                         mlx5_core_err(dev, "BUG: can't remove page at addr 0x%llx\n",
 258                                       addr);
 259                         continue;
 260                 }
 261                 dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 262                 __free_page(page);
 263         }
 264
 265 out_free:
 266         mlx5_vfree(in);
 267         return err;
 268 }
 269
 270 static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
 271                          int *nclaimed)
 272 {
 273         struct mlx5_manage_pages_inbox   in;
 274         struct mlx5_manage_pages_outbox *out;
 275         struct page *page;
 276         int num_claimed;
 277         int outlen;
 278         u64 addr;
 279         int err;
 280         int i;
 281
 282         memset(&in, 0, sizeof(in));
 283         outlen = sizeof(*out) + npages * sizeof(out->pas[0]);
 284         out = mlx5_vzalloc(outlen);
 285         if (!out)
 286                 return -ENOMEM;
 287
 288         in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
 289         in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE);
 290         in.func_id = cpu_to_be16(func_id);
 291         in.num_entries = cpu_to_be32(npages);
 292         mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen);
 293         err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
 294         if (err) {
 295                 mlx5_core_err(dev, "failed recliaming pages\n");
 296                 goto out_free;
 297         }
 298         dev->priv.fw_pages -= npages;
 299
 300         if (out->hdr.status) {
 301                 err = mlx5_cmd_status_to_err(&out->hdr);
 302                 goto out_free;
 303         }
 304
 305         num_claimed = be32_to_cpu(out->num_entries);
 306         if (nclaimed)
 307                 *nclaimed = num_claimed;
 308
 309         for (i = 0; i < num_claimed; i++) {
 310                 addr = be64_to_cpu(out->pas[i]);
 311                 page = remove_page(dev, addr);
 312                 if (!page) {
 313                         mlx5_core_warn(dev, "FW reported unknown DMA address 0x%llx\n", addr);
 314                 } else {
 315                         dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 316                         __free_page(page);
 317                 }
 318         }
 319
 320 out_free:
 321         mlx5_vfree(out);
 322         return err;
 323 }
 324
 325 static void pages_work_handler(struct work_struct *work)
 326 {
 327         struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work);
 328         struct mlx5_core_dev *dev = req->dev;
 329         int err = 0;
 330
 331         if (req->npages < 0)
 332                 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL);
 333         else if (req->npages > 0)
 334                 err = give_pages(dev, req->func_id, req->npages, 1);
 335
 336         if (err)
 337                 mlx5_core_warn(dev, "%s fail %d\n", req->npages < 0 ?
 338                                "reclaim" : "give", err);
 339
 340         kfree(req);
 341 }
 342
 343 void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id,
 344                                  s32 npages)
 345 {
 346         struct mlx5_pages_req *req;
 347
 348         req = kzalloc(sizeof(*req), GFP_ATOMIC);
 349         if (!req) {
 350                 mlx5_core_warn(dev, "failed to allocate pages request\n");
 351                 return;
 352         }
 353
 354         req->dev = dev;
 355         req->func_id = func_id;
 356         req->npages = npages;
 357         INIT_WORK(&req->work, pages_work_handler);
 358         queue_work(dev->priv.pg_wq, &req->work);
 359 }
 360
 361 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot)
 362 {
 363         u16 uninitialized_var(func_id);
 364         s32 uninitialized_var(npages);
 365         int err;
 366
 367         err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot);
 368         if (err)
 369                 return err;
 370
 371         mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n",
 372                       npages, boot ? "boot" : "init", func_id);
 373
 374         return give_pages(dev, func_id, npages, 0);
 375 }
 376
 377 static int optimal_reclaimed_pages(void)
 378 {
 379         struct mlx5_cmd_prot_block *block;
 380         struct mlx5_cmd_layout *lay;
 381         int ret;
 382
 383         ret = (sizeof(lay->in) + sizeof(block->data) -
 384                sizeof(struct mlx5_manage_pages_outbox)) / 8;
 385
 386         return ret;
 387 }
 388
 389 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
 390 {
 391         unsigned long end = jiffies + msecs_to_jiffies(5000);
 392         struct fw_page *fwp;
 393         struct rb_node *p;
 394         int err;
 395
 396         do {
 397                 p = rb_first(&dev->priv.page_root);
 398                 if (p) {
 399                         fwp = rb_entry(p, struct fw_page, rb_node);
 400                         err = reclaim_pages(dev, fwp->func_id, optimal_reclaimed_pages(), NULL);
 401                         if (err) {
 402                                 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", err);
 403                                 return err;
 404                         }
 405                 }
 406                 if (time_after(jiffies, end)) {
 407                         mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
 408                         break;
 409                 }
 410         } while (p);
 411
 412         return 0;
 413 }
 414
 415 void mlx5_pagealloc_init(struct mlx5_core_dev *dev)
 416 {
 417         dev->priv.page_root = RB_ROOT;
 418 }
 419
 420 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
 421 {
 422         /* nothing */
 423 }
 424
 425 int mlx5_pagealloc_start(struct mlx5_core_dev *dev)
 426 {
 427         dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
 428         if (!dev->priv.pg_wq)
 429                 return -ENOMEM;
 430
 431         return 0;
 432 }
 433
 434 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev)
 435 {
 436         destroy_workqueue(dev->priv.pg_wq);
 437 }