drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/pci_ids.h>
  17 #include <linux/kthread.h>
  18 #include <linux/sched.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/mutex.h>
  22 #include <linux/delay.h>
  23 #include <linux/ccp.h>
  24 #include <linux/scatterlist.h>
  25 #include <crypto/scatterwalk.h>
  26 #include <crypto/sha.h>
  27
  28 #include "ccp-dev.h"
  29
  30 enum ccp_memtype {
  31         CCP_MEMTYPE_SYSTEM = 0,
  32         CCP_MEMTYPE_KSB,
  33         CCP_MEMTYPE_LOCAL,
  34         CCP_MEMTYPE__LAST,
  35 };
  36
  37 struct ccp_dma_info {
  38         dma_addr_t address;
  39         unsigned int offset;
  40         unsigned int length;
  41         enum dma_data_direction dir;
  42 };
  43
  44 struct ccp_dm_workarea {
  45         struct device *dev;
  46         struct dma_pool *dma_pool;
  47         unsigned int length;
  48
  49         u8 *address;
  50         struct ccp_dma_info dma;
  51 };
  52
  53 struct ccp_sg_workarea {
  54         struct scatterlist *sg;
  55         int nents;
  56
  57         struct scatterlist *dma_sg;
  58         struct device *dma_dev;
  59         unsigned int dma_count;
  60         enum dma_data_direction dma_dir;
  61
  62         unsigned int sg_used;
  63
  64         u64 bytes_left;
  65 };
  66
  67 struct ccp_data {
  68         struct ccp_sg_workarea sg_wa;
  69         struct ccp_dm_workarea dm_wa;
  70 };
  71
  72 struct ccp_mem {
  73         enum ccp_memtype type;
  74         union {
  75                 struct ccp_dma_info dma;
  76                 u32 ksb;
  77         } u;
  78 };
  79
  80 struct ccp_aes_op {
  81         enum ccp_aes_type type;
  82         enum ccp_aes_mode mode;
  83         enum ccp_aes_action action;
  84 };
  85
  86 struct ccp_xts_aes_op {
  87         enum ccp_aes_action action;
  88         enum ccp_xts_aes_unit_size unit_size;
  89 };
  90
  91 struct ccp_sha_op {
  92         enum ccp_sha_type type;
  93         u64 msg_bits;
  94 };
  95
  96 struct ccp_rsa_op {
  97         u32 mod_size;
  98         u32 input_len;
  99 };
 100
 101 struct ccp_passthru_op {
 102         enum ccp_passthru_bitwise bit_mod;
 103         enum ccp_passthru_byteswap byte_swap;
 104 };
 105
 106 struct ccp_ecc_op {
 107         enum ccp_ecc_function function;
 108 };
 109
 110 struct ccp_op {
 111         struct ccp_cmd_queue *cmd_q;
 112
 113         u32 jobid;
 114         u32 ioc;
 115         u32 soc;
 116         u32 ksb_key;
 117         u32 ksb_ctx;
 118         u32 init;
 119         u32 eom;
 120
 121         struct ccp_mem src;
 122         struct ccp_mem dst;
 123
 124         union {
 125                 struct ccp_aes_op aes;
 126                 struct ccp_xts_aes_op xts;
 127                 struct ccp_sha_op sha;
 128                 struct ccp_rsa_op rsa;
 129                 struct ccp_passthru_op passthru;
 130                 struct ccp_ecc_op ecc;
 131         } u;
 132 };
 133
 134 /* SHA initial context values */
 135 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
 136         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
 137         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
 138         cpu_to_be32(SHA1_H4), 0, 0, 0,
 139 };
 140
 141 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
 142         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
 143         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
 144         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
 145         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
 146 };
 147
 148 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
 149         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
 150         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
 151         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
 152         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
 153 };
 154
 155 static u32 ccp_addr_lo(struct ccp_dma_info *info)
 156 {
 157         return lower_32_bits(info->address + info->offset);
 158 }
 159
 160 static u32 ccp_addr_hi(struct ccp_dma_info *info)
 161 {
 162         return upper_32_bits(info->address + info->offset) & 0x0000ffff;
 163 }
 164
 165 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
 166 {
 167         struct ccp_cmd_queue *cmd_q = op->cmd_q;
 168         struct ccp_device *ccp = cmd_q->ccp;
 169         void __iomem *cr_addr;
 170         u32 cr0, cmd;
 171         unsigned int i;
 172         int ret = 0;
 173
 174         /* We could read a status register to see how many free slots
 175          * are actually available, but reading that register resets it
 176          * and you could lose some error information.
 177          */
 178         cmd_q->free_slots--;
 179
 180         cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
 181               | (op->jobid << REQ0_JOBID_SHIFT)
 182               | REQ0_WAIT_FOR_WRITE;
 183
 184         if (op->soc)
 185                 cr0 |= REQ0_STOP_ON_COMPLETE
 186                        | REQ0_INT_ON_COMPLETE;
 187
 188         if (op->ioc || !cmd_q->free_slots)
 189                 cr0 |= REQ0_INT_ON_COMPLETE;
 190
 191         /* Start at CMD_REQ1 */
 192         cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
 193
 194         mutex_lock(&ccp->req_mutex);
 195
 196         /* Write CMD_REQ1 through CMD_REQx first */
 197         for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
 198                 iowrite32(*(cr + i), cr_addr);
 199
 200         /* Tell the CCP to start */
 201         wmb();
 202         iowrite32(cr0, ccp->io_regs + CMD_REQ0);
 203
 204         mutex_unlock(&ccp->req_mutex);
 205
 206         if (cr0 & REQ0_INT_ON_COMPLETE) {
 207                 /* Wait for the job to complete */
 208                 ret = wait_event_interruptible(cmd_q->int_queue,
 209                                                cmd_q->int_rcvd);
 210                 if (ret || cmd_q->cmd_error) {
 211                         /* On error delete all related jobs from the queue */
 212                         cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
 213                               | op->jobid;
 214
 215                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 216
 217                         if (!ret)
 218                                 ret = -EIO;
 219                 } else if (op->soc) {
 220                         /* Delete just head job from the queue on SoC */
 221                         cmd = DEL_Q_ACTIVE
 222                               | (cmd_q->id << DEL_Q_ID_SHIFT)
 223                               | op->jobid;
 224
 225                         iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 226                 }
 227
 228                 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
 229
 230                 cmd_q->int_rcvd = 0;
 231         }
 232
 233         return ret;
 234 }
 235
 236 static int ccp_perform_aes(struct ccp_op *op)
 237 {
 238         u32 cr[6];
 239
 240         /* Fill out the register contents for REQ1 through REQ6 */
 241         cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
 242                 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
 243                 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
 244                 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
 245                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
 246         cr[1] = op->src.u.dma.length - 1;
 247         cr[2] = ccp_addr_lo(&op->src.u.dma);
 248         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 249                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 250                 | ccp_addr_hi(&op->src.u.dma);
 251         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 252         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 253                 | ccp_addr_hi(&op->dst.u.dma);
 254
 255         if (op->u.aes.mode == CCP_AES_MODE_CFB)
 256                 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
 257
 258         if (op->eom)
 259                 cr[0] |= REQ1_EOM;
 260
 261         if (op->init)
 262                 cr[0] |= REQ1_INIT;
 263
 264         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 265 }
 266
 267 static int ccp_perform_xts_aes(struct ccp_op *op)
 268 {
 269         u32 cr[6];
 270
 271         /* Fill out the register contents for REQ1 through REQ6 */
 272         cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
 273                 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
 274                 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
 275                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
 276         cr[1] = op->src.u.dma.length - 1;
 277         cr[2] = ccp_addr_lo(&op->src.u.dma);
 278         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 279                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 280                 | ccp_addr_hi(&op->src.u.dma);
 281         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 282         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 283                 | ccp_addr_hi(&op->dst.u.dma);
 284
 285         if (op->eom)
 286                 cr[0] |= REQ1_EOM;
 287
 288         if (op->init)
 289                 cr[0] |= REQ1_INIT;
 290
 291         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 292 }
 293
 294 static int ccp_perform_sha(struct ccp_op *op)
 295 {
 296         u32 cr[6];
 297
 298         /* Fill out the register contents for REQ1 through REQ6 */
 299         cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
 300                 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
 301                 | REQ1_INIT;
 302         cr[1] = op->src.u.dma.length - 1;
 303         cr[2] = ccp_addr_lo(&op->src.u.dma);
 304         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 305                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 306                 | ccp_addr_hi(&op->src.u.dma);
 307
 308         if (op->eom) {
 309                 cr[0] |= REQ1_EOM;
 310                 cr[4] = lower_32_bits(op->u.sha.msg_bits);
 311                 cr[5] = upper_32_bits(op->u.sha.msg_bits);
 312         } else {
 313                 cr[4] = 0;
 314                 cr[5] = 0;
 315         }
 316
 317         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 318 }
 319
 320 static int ccp_perform_rsa(struct ccp_op *op)
 321 {
 322         u32 cr[6];
 323
 324         /* Fill out the register contents for REQ1 through REQ6 */
 325         cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
 326                 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
 327                 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
 328                 | REQ1_EOM;
 329         cr[1] = op->u.rsa.input_len - 1;
 330         cr[2] = ccp_addr_lo(&op->src.u.dma);
 331         cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
 332                 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 333                 | ccp_addr_hi(&op->src.u.dma);
 334         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 335         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 336                 | ccp_addr_hi(&op->dst.u.dma);
 337
 338         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 339 }
 340
 341 static int ccp_perform_passthru(struct ccp_op *op)
 342 {
 343         u32 cr[6];
 344
 345         /* Fill out the register contents for REQ1 through REQ6 */
 346         cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
 347                 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
 348                 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
 349
 350         if (op->src.type == CCP_MEMTYPE_SYSTEM)
 351                 cr[1] = op->src.u.dma.length - 1;
 352         else
 353                 cr[1] = op->dst.u.dma.length - 1;
 354
 355         if (op->src.type == CCP_MEMTYPE_SYSTEM) {
 356                 cr[2] = ccp_addr_lo(&op->src.u.dma);
 357                 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 358                         | ccp_addr_hi(&op->src.u.dma);
 359
 360                 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
 361                         cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
 362         } else {
 363                 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
 364                 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
 365         }
 366
 367         if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
 368                 cr[4] = ccp_addr_lo(&op->dst.u.dma);
 369                 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 370                         | ccp_addr_hi(&op->dst.u.dma);
 371         } else {
 372                 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
 373                 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
 374         }
 375
 376         if (op->eom)
 377                 cr[0] |= REQ1_EOM;
 378
 379         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 380 }
 381
 382 static int ccp_perform_ecc(struct ccp_op *op)
 383 {
 384         u32 cr[6];
 385
 386         /* Fill out the register contents for REQ1 through REQ6 */
 387         cr[0] = REQ1_ECC_AFFINE_CONVERT
 388                 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
 389                 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
 390                 | REQ1_EOM;
 391         cr[1] = op->src.u.dma.length - 1;
 392         cr[2] = ccp_addr_lo(&op->src.u.dma);
 393         cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 394                 | ccp_addr_hi(&op->src.u.dma);
 395         cr[4] = ccp_addr_lo(&op->dst.u.dma);
 396         cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 397                 | ccp_addr_hi(&op->dst.u.dma);
 398
 399         return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 400 }
 401
 402 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
 403 {
 404         int start;
 405
 406         for (;;) {
 407                 mutex_lock(&ccp->ksb_mutex);
 408
 409                 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
 410                                                         ccp->ksb_count,
 411                                                         ccp->ksb_start,
 412                                                         count, 0);
 413                 if (start <= ccp->ksb_count) {
 414                         bitmap_set(ccp->ksb, start, count);
 415
 416                         mutex_unlock(&ccp->ksb_mutex);
 417                         break;
 418                 }
 419
 420                 ccp->ksb_avail = 0;
 421
 422                 mutex_unlock(&ccp->ksb_mutex);
 423
 424                 /* Wait for KSB entries to become available */
 425                 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
 426                         return 0;
 427         }
 428
 429         return KSB_START + start;
 430 }
 431
 432 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
 433                          unsigned int count)
 434 {
 435         if (!start)
 436                 return;
 437
 438         mutex_lock(&ccp->ksb_mutex);
 439
 440         bitmap_clear(ccp->ksb, start - KSB_START, count);
 441
 442         ccp->ksb_avail = 1;
 443
 444         mutex_unlock(&ccp->ksb_mutex);
 445
 446         wake_up_interruptible_all(&ccp->ksb_queue);
 447 }
 448
 449 static u32 ccp_gen_jobid(struct ccp_device *ccp)
 450 {
 451         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
 452 }
 453
 454 static void ccp_sg_free(struct ccp_sg_workarea *wa)
 455 {
 456         if (wa->dma_count)
 457                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
 458
 459         wa->dma_count = 0;
 460 }
 461
 462 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
 463                                 struct scatterlist *sg, u64 len,
 464                                 enum dma_data_direction dma_dir)
 465 {
 466         memset(wa, 0, sizeof(*wa));
 467
 468         wa->sg = sg;
 469         if (!sg)
 470                 return 0;
 471
 472         wa->nents = sg_nents_for_len(sg, len);
 473         if (wa->nents < 0)
 474                 return wa->nents;
 475
 476         wa->bytes_left = len;
 477         wa->sg_used = 0;
 478
 479         if (len == 0)
 480                 return 0;
 481
 482         if (dma_dir == DMA_NONE)
 483                 return 0;
 484
 485         wa->dma_sg = sg;
 486         wa->dma_dev = dev;
 487         wa->dma_dir = dma_dir;
 488         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 489         if (!wa->dma_count)
 490                 return -ENOMEM;
 491
 492         return 0;
 493 }
 494
 495 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 496 {
 497         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 498
 499         if (!wa->sg)
 500                 return;
 501
 502         wa->sg_used += nbytes;
 503         wa->bytes_left -= nbytes;
 504         if (wa->sg_used == wa->sg->length) {
 505                 wa->sg = sg_next(wa->sg);
 506                 wa->sg_used = 0;
 507         }
 508 }
 509
 510 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 511 {
 512         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 513                 if (wa->address)
 514                         dma_pool_free(wa->dma_pool, wa->address,
 515                                       wa->dma.address);
 516         } else {
 517                 if (wa->dma.address)
 518                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 519                                          wa->dma.dir);
 520                 kfree(wa->address);
 521         }
 522
 523         wa->address = NULL;
 524         wa->dma.address = 0;
 525 }
 526
 527 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 528                                 struct ccp_cmd_queue *cmd_q,
 529                                 unsigned int len,
 530                                 enum dma_data_direction dir)
 531 {
 532         memset(wa, 0, sizeof(*wa));
 533
 534         if (!len)
 535                 return 0;
 536
 537         wa->dev = cmd_q->ccp->dev;
 538         wa->length = len;
 539
 540         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 541                 wa->dma_pool = cmd_q->dma_pool;
 542
 543                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 544                                              &wa->dma.address);
 545                 if (!wa->address)
 546                         return -ENOMEM;
 547
 548                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 549
 550                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 551         } else {
 552                 wa->address = kzalloc(len, GFP_KERNEL);
 553                 if (!wa->address)
 554                         return -ENOMEM;
 555
 556                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 557                                                  dir);
 558                 if (!wa->dma.address)
 559                         return -ENOMEM;
 560
 561                 wa->dma.length = len;
 562         }
 563         wa->dma.dir = dir;
 564
 565         return 0;
 566 }
 567
 568 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 569                             struct scatterlist *sg, unsigned int sg_offset,
 570                             unsigned int len)
 571 {
 572         WARN_ON(!wa->address);
 573
 574         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 575                                  0);
 576 }
 577
 578 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 579                             struct scatterlist *sg, unsigned int sg_offset,
 580                             unsigned int len)
 581 {
 582         WARN_ON(!wa->address);
 583
 584         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 585                                  1);
 586 }
 587
 588 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 589                                    struct scatterlist *sg,
 590                                    unsigned int len, unsigned int se_len,
 591                                    bool sign_extend)
 592 {
 593         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 594         u8 buffer[CCP_REVERSE_BUF_SIZE];
 595
 596         if (WARN_ON(se_len > sizeof(buffer)))
 597                 return -EINVAL;
 598
 599         sg_offset = len;
 600         dm_offset = 0;
 601         nbytes = len;
 602         while (nbytes) {
 603                 ksb_len = min_t(unsigned int, nbytes, se_len);
 604                 sg_offset -= ksb_len;
 605
 606                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
 607                 for (i = 0; i < ksb_len; i++)
 608                         wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
 609
 610                 dm_offset += ksb_len;
 611                 nbytes -= ksb_len;
 612
 613                 if ((ksb_len != se_len) && sign_extend) {
 614                         /* Must sign-extend to nearest sign-extend length */
 615                         if (wa->address[dm_offset - 1] & 0x80)
 616                                 memset(wa->address + dm_offset, 0xff,
 617                                        se_len - ksb_len);
 618                 }
 619         }
 620
 621         return 0;
 622 }
 623
 624 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 625                                     struct scatterlist *sg,
 626                                     unsigned int len)
 627 {
 628         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 629         u8 buffer[CCP_REVERSE_BUF_SIZE];
 630
 631         sg_offset = 0;
 632         dm_offset = len;
 633         nbytes = len;
 634         while (nbytes) {
 635                 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
 636                 dm_offset -= ksb_len;
 637
 638                 for (i = 0; i < ksb_len; i++)
 639                         buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
 640                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
 641
 642                 sg_offset += ksb_len;
 643                 nbytes -= ksb_len;
 644         }
 645 }
 646
 647 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 648 {
 649         ccp_dm_free(&data->dm_wa);
 650         ccp_sg_free(&data->sg_wa);
 651 }
 652
 653 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 654                          struct scatterlist *sg, u64 sg_len,
 655                          unsigned int dm_len,
 656                          enum dma_data_direction dir)
 657 {
 658         int ret;
 659
 660         memset(data, 0, sizeof(*data));
 661
 662         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 663                                    dir);
 664         if (ret)
 665                 goto e_err;
 666
 667         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 668         if (ret)
 669                 goto e_err;
 670
 671         return 0;
 672
 673 e_err:
 674         ccp_free_data(data, cmd_q);
 675
 676         return ret;
 677 }
 678
 679 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 680 {
 681         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 682         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 683         unsigned int buf_count, nbytes;
 684
 685         /* Clear the buffer if setting it */
 686         if (!from)
 687                 memset(dm_wa->address, 0, dm_wa->length);
 688
 689         if (!sg_wa->sg)
 690                 return 0;
 691
 692         /* Perform the copy operation
 693          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 694          *   an unsigned int
 695          */
 696         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 697         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 698                                  nbytes, from);
 699
 700         /* Update the structures and generate the count */
 701         buf_count = 0;
 702         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 703                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 704                              dm_wa->length - buf_count);
 705                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 706
 707                 buf_count += nbytes;
 708                 ccp_update_sg_workarea(sg_wa, nbytes);
 709         }
 710
 711         return buf_count;
 712 }
 713
 714 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 715 {
 716         return ccp_queue_buf(data, 0);
 717 }
 718
 719 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 720 {
 721         return ccp_queue_buf(data, 1);
 722 }
 723
 724 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 725                              struct ccp_op *op, unsigned int block_size,
 726                              bool blocksize_op)
 727 {
 728         unsigned int sg_src_len, sg_dst_len, op_len;
 729
 730         /* The CCP can only DMA from/to one address each per operation. This
 731          * requires that we find the smallest DMA area between the source
 732          * and destination. The resulting len values will always be <= UINT_MAX
 733          * because the dma length is an unsigned int.
 734          */
 735         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 736         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 737
 738         if (dst) {
 739                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 740                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 741                 op_len = min(sg_src_len, sg_dst_len);
 742         } else {
 743                 op_len = sg_src_len;
 744         }
 745
 746         /* The data operation length will be at least block_size in length
 747          * or the smaller of available sg room remaining for the source or
 748          * the destination
 749          */
 750         op_len = max(op_len, block_size);
 751
 752         /* Unless we have to buffer data, there's no reason to wait */
 753         op->soc = 0;
 754
 755         if (sg_src_len < block_size) {
 756                 /* Not enough data in the sg element, so it
 757                  * needs to be buffered into a blocksize chunk
 758                  */
 759                 int cp_len = ccp_fill_queue_buf(src);
 760
 761                 op->soc = 1;
 762                 op->src.u.dma.address = src->dm_wa.dma.address;
 763                 op->src.u.dma.offset = 0;
 764                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 765         } else {
 766                 /* Enough data in the sg element, but we need to
 767                  * adjust for any previously copied data
 768                  */
 769                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 770                 op->src.u.dma.offset = src->sg_wa.sg_used;
 771                 op->src.u.dma.length = op_len & ~(block_size - 1);
 772
 773                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 774         }
 775
 776         if (dst) {
 777                 if (sg_dst_len < block_size) {
 778                         /* Not enough room in the sg element or we're on the
 779                          * last piece of data (when using padding), so the
 780                          * output needs to be buffered into a blocksize chunk
 781                          */
 782                         op->soc = 1;
 783                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 784                         op->dst.u.dma.offset = 0;
 785                         op->dst.u.dma.length = op->src.u.dma.length;
 786                 } else {
 787                         /* Enough room in the sg element, but we need to
 788                          * adjust for any previously used area
 789                          */
 790                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 791                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 792                         op->dst.u.dma.length = op->src.u.dma.length;
 793                 }
 794         }
 795 }
 796
 797 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 798                              struct ccp_op *op)
 799 {
 800         op->init = 0;
 801
 802         if (dst) {
 803                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 804                         ccp_empty_queue_buf(dst);
 805                 else
 806                         ccp_update_sg_workarea(&dst->sg_wa,
 807                                                op->dst.u.dma.length);
 808         }
 809 }
 810
 811 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
 812                                 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 813                                 u32 byte_swap, bool from)
 814 {
 815         struct ccp_op op;
 816
 817         memset(&op, 0, sizeof(op));
 818
 819         op.cmd_q = cmd_q;
 820         op.jobid = jobid;
 821         op.eom = 1;
 822
 823         if (from) {
 824                 op.soc = 1;
 825                 op.src.type = CCP_MEMTYPE_KSB;
 826                 op.src.u.ksb = ksb;
 827                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 828                 op.dst.u.dma.address = wa->dma.address;
 829                 op.dst.u.dma.length = wa->length;
 830         } else {
 831                 op.src.type = CCP_MEMTYPE_SYSTEM;
 832                 op.src.u.dma.address = wa->dma.address;
 833                 op.src.u.dma.length = wa->length;
 834                 op.dst.type = CCP_MEMTYPE_KSB;
 835                 op.dst.u.ksb = ksb;
 836         }
 837
 838         op.u.passthru.byte_swap = byte_swap;
 839
 840         return ccp_perform_passthru(&op);
 841 }
 842
 843 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
 844                            struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 845                            u32 byte_swap)
 846 {
 847         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
 848 }
 849
 850 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
 851                              struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 852                              u32 byte_swap)
 853 {
 854         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
 855 }
 856
 857 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 858                                 struct ccp_cmd *cmd)
 859 {
 860         struct ccp_aes_engine *aes = &cmd->u.aes;
 861         struct ccp_dm_workarea key, ctx;
 862         struct ccp_data src;
 863         struct ccp_op op;
 864         unsigned int dm_offset;
 865         int ret;
 866
 867         if (!((aes->key_len == AES_KEYSIZE_128) ||
 868               (aes->key_len == AES_KEYSIZE_192) ||
 869               (aes->key_len == AES_KEYSIZE_256)))
 870                 return -EINVAL;
 871
 872         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 873                 return -EINVAL;
 874
 875         if (aes->iv_len != AES_BLOCK_SIZE)
 876                 return -EINVAL;
 877
 878         if (!aes->key || !aes->iv || !aes->src)
 879                 return -EINVAL;
 880
 881         if (aes->cmac_final) {
 882                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 883                         return -EINVAL;
 884
 885                 if (!aes->cmac_key)
 886                         return -EINVAL;
 887         }
 888
 889         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
 890         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
 891
 892         ret = -EIO;
 893         memset(&op, 0, sizeof(op));
 894         op.cmd_q = cmd_q;
 895         op.jobid = ccp_gen_jobid(cmd_q->ccp);
 896         op.ksb_key = cmd_q->ksb_key;
 897         op.ksb_ctx = cmd_q->ksb_ctx;
 898         op.init = 1;
 899         op.u.aes.type = aes->type;
 900         op.u.aes.mode = aes->mode;
 901         op.u.aes.action = aes->action;
 902
 903         /* All supported key sizes fit in a single (32-byte) KSB entry
 904          * and must be in little endian format. Use the 256-bit byte
 905          * swap passthru option to convert from big endian to little
 906          * endian.
 907          */
 908         ret = ccp_init_dm_workarea(&key, cmd_q,
 909                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
 910                                    DMA_TO_DEVICE);
 911         if (ret)
 912                 return ret;
 913
 914         dm_offset = CCP_KSB_BYTES - aes->key_len;
 915         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 916         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
 917                               CCP_PASSTHRU_BYTESWAP_256BIT);
 918         if (ret) {
 919                 cmd->engine_error = cmd_q->cmd_error;
 920                 goto e_key;
 921         }
 922
 923         /* The AES context fits in a single (32-byte) KSB entry and
 924          * must be in little endian format. Use the 256-bit byte swap
 925          * passthru option to convert from big endian to little endian.
 926          */
 927         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 928                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
 929                                    DMA_BIDIRECTIONAL);
 930         if (ret)
 931                 goto e_key;
 932
 933         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 934         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 935         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 936                               CCP_PASSTHRU_BYTESWAP_256BIT);
 937         if (ret) {
 938                 cmd->engine_error = cmd_q->cmd_error;
 939                 goto e_ctx;
 940         }
 941
 942         /* Send data to the CCP AES engine */
 943         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 944                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 945         if (ret)
 946                 goto e_ctx;
 947
 948         while (src.sg_wa.bytes_left) {
 949                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 950                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 951                         op.eom = 1;
 952
 953                         /* Push the K1/K2 key to the CCP now */
 954                         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
 955                                                 op.ksb_ctx,
 956                                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 957                         if (ret) {
 958                                 cmd->engine_error = cmd_q->cmd_error;
 959                                 goto e_src;
 960                         }
 961
 962                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 963                                         aes->cmac_key_len);
 964                         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 965                                               CCP_PASSTHRU_BYTESWAP_256BIT);
 966                         if (ret) {
 967                                 cmd->engine_error = cmd_q->cmd_error;
 968                                 goto e_src;
 969                         }
 970                 }
 971
 972                 ret = ccp_perform_aes(&op);
 973                 if (ret) {
 974                         cmd->engine_error = cmd_q->cmd_error;
 975                         goto e_src;
 976                 }
 977
 978                 ccp_process_data(&src, NULL, &op);
 979         }
 980
 981         /* Retrieve the AES context - convert from LE to BE using
 982          * 32-byte (256-bit) byteswapping
 983          */
 984         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 985                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 986         if (ret) {
 987                 cmd->engine_error = cmd_q->cmd_error;
 988                 goto e_src;
 989         }
 990
 991         /* ...but we only need AES_BLOCK_SIZE bytes */
 992         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 993         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 994
 995 e_src:
 996         ccp_free_data(&src, cmd_q);
 997
 998 e_ctx:
 999         ccp_dm_free(&ctx);
1000
1001 e_key:
1002         ccp_dm_free(&key);
1003
1004         return ret;
1005 }
1006
1007 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1008 {
1009         struct ccp_aes_engine *aes = &cmd->u.aes;
1010         struct ccp_dm_workarea key, ctx;
1011         struct ccp_data src, dst;
1012         struct ccp_op op;
1013         unsigned int dm_offset;
1014         bool in_place = false;
1015         int ret;
1016
1017         if (aes->mode == CCP_AES_MODE_CMAC)
1018                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1019
1020         if (!((aes->key_len == AES_KEYSIZE_128) ||
1021               (aes->key_len == AES_KEYSIZE_192) ||
1022               (aes->key_len == AES_KEYSIZE_256)))
1023                 return -EINVAL;
1024
1025         if (((aes->mode == CCP_AES_MODE_ECB) ||
1026              (aes->mode == CCP_AES_MODE_CBC) ||
1027              (aes->mode == CCP_AES_MODE_CFB)) &&
1028             (aes->src_len & (AES_BLOCK_SIZE - 1)))
1029                 return -EINVAL;
1030
1031         if (!aes->key || !aes->src || !aes->dst)
1032                 return -EINVAL;
1033
1034         if (aes->mode != CCP_AES_MODE_ECB) {
1035                 if (aes->iv_len != AES_BLOCK_SIZE)
1036                         return -EINVAL;
1037
1038                 if (!aes->iv)
1039                         return -EINVAL;
1040         }
1041
1042         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1043         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1044
1045         ret = -EIO;
1046         memset(&op, 0, sizeof(op));
1047         op.cmd_q = cmd_q;
1048         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1049         op.ksb_key = cmd_q->ksb_key;
1050         op.ksb_ctx = cmd_q->ksb_ctx;
1051         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1052         op.u.aes.type = aes->type;
1053         op.u.aes.mode = aes->mode;
1054         op.u.aes.action = aes->action;
1055
1056         /* All supported key sizes fit in a single (32-byte) KSB entry
1057          * and must be in little endian format. Use the 256-bit byte
1058          * swap passthru option to convert from big endian to little
1059          * endian.
1060          */
1061         ret = ccp_init_dm_workarea(&key, cmd_q,
1062                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1063                                    DMA_TO_DEVICE);
1064         if (ret)
1065                 return ret;
1066
1067         dm_offset = CCP_KSB_BYTES - aes->key_len;
1068         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1069         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1070                               CCP_PASSTHRU_BYTESWAP_256BIT);
1071         if (ret) {
1072                 cmd->engine_error = cmd_q->cmd_error;
1073                 goto e_key;
1074         }
1075
1076         /* The AES context fits in a single (32-byte) KSB entry and
1077          * must be in little endian format. Use the 256-bit byte swap
1078          * passthru option to convert from big endian to little endian.
1079          */
1080         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1081                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1082                                    DMA_BIDIRECTIONAL);
1083         if (ret)
1084                 goto e_key;
1085
1086         if (aes->mode != CCP_AES_MODE_ECB) {
1087                 /* Load the AES context - conver to LE */
1088                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1089                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1090                 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1091                                       CCP_PASSTHRU_BYTESWAP_256BIT);
1092                 if (ret) {
1093                         cmd->engine_error = cmd_q->cmd_error;
1094                         goto e_ctx;
1095                 }
1096         }
1097
1098         /* Prepare the input and output data workareas. For in-place
1099          * operations we need to set the dma direction to BIDIRECTIONAL
1100          * and copy the src workarea to the dst workarea.
1101          */
1102         if (sg_virt(aes->src) == sg_virt(aes->dst))
1103                 in_place = true;
1104
1105         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1106                             AES_BLOCK_SIZE,
1107                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1108         if (ret)
1109                 goto e_ctx;
1110
1111         if (in_place) {
1112                 dst = src;
1113         } else {
1114                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1115                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1116                 if (ret)
1117                         goto e_src;
1118         }
1119
1120         /* Send data to the CCP AES engine */
1121         while (src.sg_wa.bytes_left) {
1122                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1123                 if (!src.sg_wa.bytes_left) {
1124                         op.eom = 1;
1125
1126                         /* Since we don't retrieve the AES context in ECB
1127                          * mode we have to wait for the operation to complete
1128                          * on the last piece of data
1129                          */
1130                         if (aes->mode == CCP_AES_MODE_ECB)
1131                                 op.soc = 1;
1132                 }
1133
1134                 ret = ccp_perform_aes(&op);
1135                 if (ret) {
1136                         cmd->engine_error = cmd_q->cmd_error;
1137                         goto e_dst;
1138                 }
1139
1140                 ccp_process_data(&src, &dst, &op);
1141         }
1142
1143         if (aes->mode != CCP_AES_MODE_ECB) {
1144                 /* Retrieve the AES context - convert from LE to BE using
1145                  * 32-byte (256-bit) byteswapping
1146                  */
1147                 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1148                                         CCP_PASSTHRU_BYTESWAP_256BIT);
1149                 if (ret) {
1150                         cmd->engine_error = cmd_q->cmd_error;
1151                         goto e_dst;
1152                 }
1153
1154                 /* ...but we only need AES_BLOCK_SIZE bytes */
1155                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1156                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1157         }
1158
1159 e_dst:
1160         if (!in_place)
1161                 ccp_free_data(&dst, cmd_q);
1162
1163 e_src:
1164         ccp_free_data(&src, cmd_q);
1165
1166 e_ctx:
1167         ccp_dm_free(&ctx);
1168
1169 e_key:
1170         ccp_dm_free(&key);
1171
1172         return ret;
1173 }
1174
1175 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1176                                struct ccp_cmd *cmd)
1177 {
1178         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1179         struct ccp_dm_workarea key, ctx;
1180         struct ccp_data src, dst;
1181         struct ccp_op op;
1182         unsigned int unit_size, dm_offset;
1183         bool in_place = false;
1184         int ret;
1185
1186         switch (xts->unit_size) {
1187         case CCP_XTS_AES_UNIT_SIZE_16:
1188                 unit_size = 16;
1189                 break;
1190         case CCP_XTS_AES_UNIT_SIZE_512:
1191                 unit_size = 512;
1192                 break;
1193         case CCP_XTS_AES_UNIT_SIZE_1024:
1194                 unit_size = 1024;
1195                 break;
1196         case CCP_XTS_AES_UNIT_SIZE_2048:
1197                 unit_size = 2048;
1198                 break;
1199         case CCP_XTS_AES_UNIT_SIZE_4096:
1200                 unit_size = 4096;
1201                 break;
1202
1203         default:
1204                 return -EINVAL;
1205         }
1206
1207         if (xts->key_len != AES_KEYSIZE_128)
1208                 return -EINVAL;
1209
1210         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1211                 return -EINVAL;
1212
1213         if (xts->iv_len != AES_BLOCK_SIZE)
1214                 return -EINVAL;
1215
1216         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1217                 return -EINVAL;
1218
1219         BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1220         BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1221
1222         ret = -EIO;
1223         memset(&op, 0, sizeof(op));
1224         op.cmd_q = cmd_q;
1225         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1226         op.ksb_key = cmd_q->ksb_key;
1227         op.ksb_ctx = cmd_q->ksb_ctx;
1228         op.init = 1;
1229         op.u.xts.action = xts->action;
1230         op.u.xts.unit_size = xts->unit_size;
1231
1232         /* All supported key sizes fit in a single (32-byte) KSB entry
1233          * and must be in little endian format. Use the 256-bit byte
1234          * swap passthru option to convert from big endian to little
1235          * endian.
1236          */
1237         ret = ccp_init_dm_workarea(&key, cmd_q,
1238                                    CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1239                                    DMA_TO_DEVICE);
1240         if (ret)
1241                 return ret;
1242
1243         dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1244         ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1245         ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1246         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1247                               CCP_PASSTHRU_BYTESWAP_256BIT);
1248         if (ret) {
1249                 cmd->engine_error = cmd_q->cmd_error;
1250                 goto e_key;
1251         }
1252
1253         /* The AES context fits in a single (32-byte) KSB entry and
1254          * for XTS is already in little endian format so no byte swapping
1255          * is needed.
1256          */
1257         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1258                                    CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1259                                    DMA_BIDIRECTIONAL);
1260         if (ret)
1261                 goto e_key;
1262
1263         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1264         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1265                               CCP_PASSTHRU_BYTESWAP_NOOP);
1266         if (ret) {
1267                 cmd->engine_error = cmd_q->cmd_error;
1268                 goto e_ctx;
1269         }
1270
1271         /* Prepare the input and output data workareas. For in-place
1272          * operations we need to set the dma direction to BIDIRECTIONAL
1273          * and copy the src workarea to the dst workarea.
1274          */
1275         if (sg_virt(xts->src) == sg_virt(xts->dst))
1276                 in_place = true;
1277
1278         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1279                             unit_size,
1280                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1281         if (ret)
1282                 goto e_ctx;
1283
1284         if (in_place) {
1285                 dst = src;
1286         } else {
1287                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1288                                     unit_size, DMA_FROM_DEVICE);
1289                 if (ret)
1290                         goto e_src;
1291         }
1292
1293         /* Send data to the CCP AES engine */
1294         while (src.sg_wa.bytes_left) {
1295                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1296                 if (!src.sg_wa.bytes_left)
1297                         op.eom = 1;
1298
1299                 ret = ccp_perform_xts_aes(&op);
1300                 if (ret) {
1301                         cmd->engine_error = cmd_q->cmd_error;
1302                         goto e_dst;
1303                 }
1304
1305                 ccp_process_data(&src, &dst, &op);
1306         }
1307
1308         /* Retrieve the AES context - convert from LE to BE using
1309          * 32-byte (256-bit) byteswapping
1310          */
1311         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1312                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1313         if (ret) {
1314                 cmd->engine_error = cmd_q->cmd_error;
1315                 goto e_dst;
1316         }
1317
1318         /* ...but we only need AES_BLOCK_SIZE bytes */
1319         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1320         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1321
1322 e_dst:
1323         if (!in_place)
1324                 ccp_free_data(&dst, cmd_q);
1325
1326 e_src:
1327         ccp_free_data(&src, cmd_q);
1328
1329 e_ctx:
1330         ccp_dm_free(&ctx);
1331
1332 e_key:
1333         ccp_dm_free(&key);
1334
1335         return ret;
1336 }
1337
1338 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1339 {
1340         struct ccp_sha_engine *sha = &cmd->u.sha;
1341         struct ccp_dm_workarea ctx;
1342         struct ccp_data src;
1343         struct ccp_op op;
1344         int ret;
1345
1346         if (sha->ctx_len != CCP_SHA_CTXSIZE)
1347                 return -EINVAL;
1348
1349         if (!sha->ctx)
1350                 return -EINVAL;
1351
1352         if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1353                 return -EINVAL;
1354
1355         if (!sha->src_len) {
1356                 const u8 *sha_zero;
1357
1358                 /* Not final, just return */
1359                 if (!sha->final)
1360                         return 0;
1361
1362                 /* CCP can't do a zero length sha operation so the caller
1363                  * must buffer the data.
1364                  */
1365                 if (sha->msg_bits)
1366                         return -EINVAL;
1367
1368                 /* The CCP cannot perform zero-length sha operations so the
1369                  * caller is required to buffer data for the final operation.
1370                  * However, a sha operation for a message with a total length
1371                  * of zero is valid so known values are required to supply
1372                  * the result.
1373                  */
1374                 switch (sha->type) {
1375                 case CCP_SHA_TYPE_1:
1376                         sha_zero = sha1_zero_message_hash;
1377                         break;
1378                 case CCP_SHA_TYPE_224:
1379                         sha_zero = sha224_zero_message_hash;
1380                         break;
1381                 case CCP_SHA_TYPE_256:
1382                         sha_zero = sha256_zero_message_hash;
1383                         break;
1384                 default:
1385                         return -EINVAL;
1386                 }
1387
1388                 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1389                                          sha->ctx_len, 1);
1390
1391                 return 0;
1392         }
1393
1394         if (!sha->src)
1395                 return -EINVAL;
1396
1397         BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1398
1399         memset(&op, 0, sizeof(op));
1400         op.cmd_q = cmd_q;
1401         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1402         op.ksb_ctx = cmd_q->ksb_ctx;
1403         op.u.sha.type = sha->type;
1404         op.u.sha.msg_bits = sha->msg_bits;
1405
1406         /* The SHA context fits in a single (32-byte) KSB entry and
1407          * must be in little endian format. Use the 256-bit byte swap
1408          * passthru option to convert from big endian to little endian.
1409          */
1410         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1411                                    CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1412                                    DMA_BIDIRECTIONAL);
1413         if (ret)
1414                 return ret;
1415
1416         if (sha->first) {
1417                 const __be32 *init;
1418
1419                 switch (sha->type) {
1420                 case CCP_SHA_TYPE_1:
1421                         init = ccp_sha1_init;
1422                         break;
1423                 case CCP_SHA_TYPE_224:
1424                         init = ccp_sha224_init;
1425                         break;
1426                 case CCP_SHA_TYPE_256:
1427                         init = ccp_sha256_init;
1428                         break;
1429                 default:
1430                         ret = -EINVAL;
1431                         goto e_ctx;
1432                 }
1433                 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
1434         } else {
1435                 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1436         }
1437
1438         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1439                               CCP_PASSTHRU_BYTESWAP_256BIT);
1440         if (ret) {
1441                 cmd->engine_error = cmd_q->cmd_error;
1442                 goto e_ctx;
1443         }
1444
1445         /* Send data to the CCP SHA engine */
1446         ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1447                             CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1448         if (ret)
1449                 goto e_ctx;
1450
1451         while (src.sg_wa.bytes_left) {
1452                 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1453                 if (sha->final && !src.sg_wa.bytes_left)
1454                         op.eom = 1;
1455
1456                 ret = ccp_perform_sha(&op);
1457                 if (ret) {
1458                         cmd->engine_error = cmd_q->cmd_error;
1459                         goto e_data;
1460                 }
1461
1462                 ccp_process_data(&src, NULL, &op);
1463         }
1464
1465         /* Retrieve the SHA context - convert from LE to BE using
1466          * 32-byte (256-bit) byteswapping to BE
1467          */
1468         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1469                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1470         if (ret) {
1471                 cmd->engine_error = cmd_q->cmd_error;
1472                 goto e_data;
1473         }
1474
1475         ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1476
1477         if (sha->final && sha->opad) {
1478                 /* HMAC operation, recursively perform final SHA */
1479                 struct ccp_cmd hmac_cmd;
1480                 struct scatterlist sg;
1481                 u64 block_size, digest_size;
1482                 u8 *hmac_buf;
1483
1484                 switch (sha->type) {
1485                 case CCP_SHA_TYPE_1:
1486                         block_size = SHA1_BLOCK_SIZE;
1487                         digest_size = SHA1_DIGEST_SIZE;
1488                         break;
1489                 case CCP_SHA_TYPE_224:
1490                         block_size = SHA224_BLOCK_SIZE;
1491                         digest_size = SHA224_DIGEST_SIZE;
1492                         break;
1493                 case CCP_SHA_TYPE_256:
1494                         block_size = SHA256_BLOCK_SIZE;
1495                         digest_size = SHA256_DIGEST_SIZE;
1496                         break;
1497                 default:
1498                         ret = -EINVAL;
1499                         goto e_data;
1500                 }
1501
1502                 if (sha->opad_len != block_size) {
1503                         ret = -EINVAL;
1504                         goto e_data;
1505                 }
1506
1507                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1508                 if (!hmac_buf) {
1509                         ret = -ENOMEM;
1510                         goto e_data;
1511                 }
1512                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1513
1514                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1515                 memcpy(hmac_buf + block_size, ctx.address, digest_size);
1516
1517                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1518                 hmac_cmd.engine = CCP_ENGINE_SHA;
1519                 hmac_cmd.u.sha.type = sha->type;
1520                 hmac_cmd.u.sha.ctx = sha->ctx;
1521                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1522                 hmac_cmd.u.sha.src = &sg;
1523                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1524                 hmac_cmd.u.sha.opad = NULL;
1525                 hmac_cmd.u.sha.opad_len = 0;
1526                 hmac_cmd.u.sha.first = 1;
1527                 hmac_cmd.u.sha.final = 1;
1528                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1529
1530                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1531                 if (ret)
1532                         cmd->engine_error = hmac_cmd.engine_error;
1533
1534                 kfree(hmac_buf);
1535         }
1536
1537 e_data:
1538         ccp_free_data(&src, cmd_q);
1539
1540 e_ctx:
1541         ccp_dm_free(&ctx);
1542
1543         return ret;
1544 }
1545
1546 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1547 {
1548         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1549         struct ccp_dm_workarea exp, src;
1550         struct ccp_data dst;
1551         struct ccp_op op;
1552         unsigned int ksb_count, i_len, o_len;
1553         int ret;
1554
1555         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1556                 return -EINVAL;
1557
1558         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1559                 return -EINVAL;
1560
1561         /* The RSA modulus must precede the message being acted upon, so
1562          * it must be copied to a DMA area where the message and the
1563          * modulus can be concatenated.  Therefore the input buffer
1564          * length required is twice the output buffer length (which
1565          * must be a multiple of 256-bits).
1566          */
1567         o_len = ((rsa->key_size + 255) / 256) * 32;
1568         i_len = o_len * 2;
1569
1570         ksb_count = o_len / CCP_KSB_BYTES;
1571
1572         memset(&op, 0, sizeof(op));
1573         op.cmd_q = cmd_q;
1574         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1575         op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1576         if (!op.ksb_key)
1577                 return -EIO;
1578
1579         /* The RSA exponent may span multiple (32-byte) KSB entries and must
1580          * be in little endian format. Reverse copy each 32-byte chunk
1581          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1582          * and each byte within that chunk and do not perform any byte swap
1583          * operations on the passthru operation.
1584          */
1585         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1586         if (ret)
1587                 goto e_ksb;
1588
1589         ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1590                                       CCP_KSB_BYTES, false);
1591         if (ret)
1592                 goto e_exp;
1593         ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1594                               CCP_PASSTHRU_BYTESWAP_NOOP);
1595         if (ret) {
1596                 cmd->engine_error = cmd_q->cmd_error;
1597                 goto e_exp;
1598         }
1599
1600         /* Concatenate the modulus and the message. Both the modulus and
1601          * the operands must be in little endian format.  Since the input
1602          * is in big endian format it must be converted.
1603          */
1604         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1605         if (ret)
1606                 goto e_exp;
1607
1608         ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1609                                       CCP_KSB_BYTES, false);
1610         if (ret)
1611                 goto e_src;
1612         src.address += o_len;   /* Adjust the address for the copy operation */
1613         ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1614                                       CCP_KSB_BYTES, false);
1615         if (ret)
1616                 goto e_src;
1617         src.address -= o_len;   /* Reset the address to original value */
1618
1619         /* Prepare the output area for the operation */
1620         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1621                             o_len, DMA_FROM_DEVICE);
1622         if (ret)
1623                 goto e_src;
1624
1625         op.soc = 1;
1626         op.src.u.dma.address = src.dma.address;
1627         op.src.u.dma.offset = 0;
1628         op.src.u.dma.length = i_len;
1629         op.dst.u.dma.address = dst.dm_wa.dma.address;
1630         op.dst.u.dma.offset = 0;
1631         op.dst.u.dma.length = o_len;
1632
1633         op.u.rsa.mod_size = rsa->key_size;
1634         op.u.rsa.input_len = i_len;
1635
1636         ret = ccp_perform_rsa(&op);
1637         if (ret) {
1638                 cmd->engine_error = cmd_q->cmd_error;
1639                 goto e_dst;
1640         }
1641
1642         ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1643
1644 e_dst:
1645         ccp_free_data(&dst, cmd_q);
1646
1647 e_src:
1648         ccp_dm_free(&src);
1649
1650 e_exp:
1651         ccp_dm_free(&exp);
1652
1653 e_ksb:
1654         ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1655
1656         return ret;
1657 }
1658
1659 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1660                                 struct ccp_cmd *cmd)
1661 {
1662         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1663         struct ccp_dm_workarea mask;
1664         struct ccp_data src, dst;
1665         struct ccp_op op;
1666         bool in_place = false;
1667         unsigned int i;
1668         int ret;
1669
1670         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1671                 return -EINVAL;
1672
1673         if (!pt->src || !pt->dst)
1674                 return -EINVAL;
1675
1676         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1677                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1678                         return -EINVAL;
1679                 if (!pt->mask)
1680                         return -EINVAL;
1681         }
1682
1683         BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1684
1685         memset(&op, 0, sizeof(op));
1686         op.cmd_q = cmd_q;
1687         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1688
1689         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1690                 /* Load the mask */
1691                 op.ksb_key = cmd_q->ksb_key;
1692
1693                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1694                                            CCP_PASSTHRU_KSB_COUNT *
1695                                            CCP_KSB_BYTES,
1696                                            DMA_TO_DEVICE);
1697                 if (ret)
1698                         return ret;
1699
1700                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1701                 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1702                                       CCP_PASSTHRU_BYTESWAP_NOOP);
1703                 if (ret) {
1704                         cmd->engine_error = cmd_q->cmd_error;
1705                         goto e_mask;
1706                 }
1707         }
1708
1709         /* Prepare the input and output data workareas. For in-place
1710          * operations we need to set the dma direction to BIDIRECTIONAL
1711          * and copy the src workarea to the dst workarea.
1712          */
1713         if (sg_virt(pt->src) == sg_virt(pt->dst))
1714                 in_place = true;
1715
1716         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1717                             CCP_PASSTHRU_MASKSIZE,
1718                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1719         if (ret)
1720                 goto e_mask;
1721
1722         if (in_place) {
1723                 dst = src;
1724         } else {
1725                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1726                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1727                 if (ret)
1728                         goto e_src;
1729         }
1730
1731         /* Send data to the CCP Passthru engine
1732          *   Because the CCP engine works on a single source and destination
1733          *   dma address at a time, each entry in the source scatterlist
1734          *   (after the dma_map_sg call) must be less than or equal to the
1735          *   (remaining) length in the destination scatterlist entry and the
1736          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1737          */
1738         dst.sg_wa.sg_used = 0;
1739         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1740                 if (!dst.sg_wa.sg ||
1741                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1742                         ret = -EINVAL;
1743                         goto e_dst;
1744                 }
1745
1746                 if (i == src.sg_wa.dma_count) {
1747                         op.eom = 1;
1748                         op.soc = 1;
1749                 }
1750
1751                 op.src.type = CCP_MEMTYPE_SYSTEM;
1752                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1753                 op.src.u.dma.offset = 0;
1754                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1755
1756                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1757                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1758                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1759                 op.dst.u.dma.length = op.src.u.dma.length;
1760
1761                 ret = ccp_perform_passthru(&op);
1762                 if (ret) {
1763                         cmd->engine_error = cmd_q->cmd_error;
1764                         goto e_dst;
1765                 }
1766
1767                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1768                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1769                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1770                         dst.sg_wa.sg_used = 0;
1771                 }
1772                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1773         }
1774
1775 e_dst:
1776         if (!in_place)
1777                 ccp_free_data(&dst, cmd_q);
1778
1779 e_src:
1780         ccp_free_data(&src, cmd_q);
1781
1782 e_mask:
1783         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1784                 ccp_dm_free(&mask);
1785
1786         return ret;
1787 }
1788
1789 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1790 {
1791         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1792         struct ccp_dm_workarea src, dst;
1793         struct ccp_op op;
1794         int ret;
1795         u8 *save;
1796
1797         if (!ecc->u.mm.operand_1 ||
1798             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1799                 return -EINVAL;
1800
1801         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1802                 if (!ecc->u.mm.operand_2 ||
1803                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1804                         return -EINVAL;
1805
1806         if (!ecc->u.mm.result ||
1807             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1808                 return -EINVAL;
1809
1810         memset(&op, 0, sizeof(op));
1811         op.cmd_q = cmd_q;
1812         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1813
1814         /* Concatenate the modulus and the operands. Both the modulus and
1815          * the operands must be in little endian format.  Since the input
1816          * is in big endian format it must be converted and placed in a
1817          * fixed length buffer.
1818          */
1819         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1820                                    DMA_TO_DEVICE);
1821         if (ret)
1822                 return ret;
1823
1824         /* Save the workarea address since it is updated in order to perform
1825          * the concatenation
1826          */
1827         save = src.address;
1828
1829         /* Copy the ECC modulus */
1830         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1831                                       CCP_ECC_OPERAND_SIZE, false);
1832         if (ret)
1833                 goto e_src;
1834         src.address += CCP_ECC_OPERAND_SIZE;
1835
1836         /* Copy the first operand */
1837         ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1838                                       ecc->u.mm.operand_1_len,
1839                                       CCP_ECC_OPERAND_SIZE, false);
1840         if (ret)
1841                 goto e_src;
1842         src.address += CCP_ECC_OPERAND_SIZE;
1843
1844         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1845                 /* Copy the second operand */
1846                 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1847                                               ecc->u.mm.operand_2_len,
1848                                               CCP_ECC_OPERAND_SIZE, false);
1849                 if (ret)
1850                         goto e_src;
1851                 src.address += CCP_ECC_OPERAND_SIZE;
1852         }
1853
1854         /* Restore the workarea address */
1855         src.address = save;
1856
1857         /* Prepare the output area for the operation */
1858         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1859                                    DMA_FROM_DEVICE);
1860         if (ret)
1861                 goto e_src;
1862
1863         op.soc = 1;
1864         op.src.u.dma.address = src.dma.address;
1865         op.src.u.dma.offset = 0;
1866         op.src.u.dma.length = src.length;
1867         op.dst.u.dma.address = dst.dma.address;
1868         op.dst.u.dma.offset = 0;
1869         op.dst.u.dma.length = dst.length;
1870
1871         op.u.ecc.function = cmd->u.ecc.function;
1872
1873         ret = ccp_perform_ecc(&op);
1874         if (ret) {
1875                 cmd->engine_error = cmd_q->cmd_error;
1876                 goto e_dst;
1877         }
1878
1879         ecc->ecc_result = le16_to_cpup(
1880                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1881         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1882                 ret = -EIO;
1883                 goto e_dst;
1884         }
1885
1886         /* Save the ECC result */
1887         ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1888
1889 e_dst:
1890         ccp_dm_free(&dst);
1891
1892 e_src:
1893         ccp_dm_free(&src);
1894
1895         return ret;
1896 }
1897
1898 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1899 {
1900         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1901         struct ccp_dm_workarea src, dst;
1902         struct ccp_op op;
1903         int ret;
1904         u8 *save;
1905
1906         if (!ecc->u.pm.point_1.x ||
1907             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1908             !ecc->u.pm.point_1.y ||
1909             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1910                 return -EINVAL;
1911
1912         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1913                 if (!ecc->u.pm.point_2.x ||
1914                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1915                     !ecc->u.pm.point_2.y ||
1916                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1917                         return -EINVAL;
1918         } else {
1919                 if (!ecc->u.pm.domain_a ||
1920                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1921                         return -EINVAL;
1922
1923                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1924                         if (!ecc->u.pm.scalar ||
1925                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1926                                 return -EINVAL;
1927         }
1928
1929         if (!ecc->u.pm.result.x ||
1930             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1931             !ecc->u.pm.result.y ||
1932             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1933                 return -EINVAL;
1934
1935         memset(&op, 0, sizeof(op));
1936         op.cmd_q = cmd_q;
1937         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1938
1939         /* Concatenate the modulus and the operands. Both the modulus and
1940          * the operands must be in little endian format.  Since the input
1941          * is in big endian format it must be converted and placed in a
1942          * fixed length buffer.
1943          */
1944         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1945                                    DMA_TO_DEVICE);
1946         if (ret)
1947                 return ret;
1948
1949         /* Save the workarea address since it is updated in order to perform
1950          * the concatenation
1951          */
1952         save = src.address;
1953
1954         /* Copy the ECC modulus */
1955         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1956                                       CCP_ECC_OPERAND_SIZE, false);
1957         if (ret)
1958                 goto e_src;
1959         src.address += CCP_ECC_OPERAND_SIZE;
1960
1961         /* Copy the first point X and Y coordinate */
1962         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1963                                       ecc->u.pm.point_1.x_len,
1964                                       CCP_ECC_OPERAND_SIZE, false);
1965         if (ret)
1966                 goto e_src;
1967         src.address += CCP_ECC_OPERAND_SIZE;
1968         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1969                                       ecc->u.pm.point_1.y_len,
1970                                       CCP_ECC_OPERAND_SIZE, false);
1971         if (ret)
1972                 goto e_src;
1973         src.address += CCP_ECC_OPERAND_SIZE;
1974
1975         /* Set the first point Z coordianate to 1 */
1976         *src.address = 0x01;
1977         src.address += CCP_ECC_OPERAND_SIZE;
1978
1979         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1980                 /* Copy the second point X and Y coordinate */
1981                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1982                                               ecc->u.pm.point_2.x_len,
1983                                               CCP_ECC_OPERAND_SIZE, false);
1984                 if (ret)
1985                         goto e_src;
1986                 src.address += CCP_ECC_OPERAND_SIZE;
1987                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1988                                               ecc->u.pm.point_2.y_len,
1989                                               CCP_ECC_OPERAND_SIZE, false);
1990                 if (ret)
1991                         goto e_src;
1992                 src.address += CCP_ECC_OPERAND_SIZE;
1993
1994                 /* Set the second point Z coordianate to 1 */
1995                 *src.address = 0x01;
1996                 src.address += CCP_ECC_OPERAND_SIZE;
1997         } else {
1998                 /* Copy the Domain "a" parameter */
1999                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
2000                                               ecc->u.pm.domain_a_len,
2001                                               CCP_ECC_OPERAND_SIZE, false);
2002                 if (ret)
2003                         goto e_src;
2004                 src.address += CCP_ECC_OPERAND_SIZE;
2005
2006                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2007                         /* Copy the scalar value */
2008                         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
2009                                                       ecc->u.pm.scalar_len,
2010                                                       CCP_ECC_OPERAND_SIZE,
2011                                                       false);
2012                         if (ret)
2013                                 goto e_src;
2014                         src.address += CCP_ECC_OPERAND_SIZE;
2015                 }
2016         }
2017
2018         /* Restore the workarea address */
2019         src.address = save;
2020
2021         /* Prepare the output area for the operation */
2022         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2023                                    DMA_FROM_DEVICE);
2024         if (ret)
2025                 goto e_src;
2026
2027         op.soc = 1;
2028         op.src.u.dma.address = src.dma.address;
2029         op.src.u.dma.offset = 0;
2030         op.src.u.dma.length = src.length;
2031         op.dst.u.dma.address = dst.dma.address;
2032         op.dst.u.dma.offset = 0;
2033         op.dst.u.dma.length = dst.length;
2034
2035         op.u.ecc.function = cmd->u.ecc.function;
2036
2037         ret = ccp_perform_ecc(&op);
2038         if (ret) {
2039                 cmd->engine_error = cmd_q->cmd_error;
2040                 goto e_dst;
2041         }
2042
2043         ecc->ecc_result = le16_to_cpup(
2044                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2045         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2046                 ret = -EIO;
2047                 goto e_dst;
2048         }
2049
2050         /* Save the workarea address since it is updated as we walk through
2051          * to copy the point math result
2052          */
2053         save = dst.address;
2054
2055         /* Save the ECC result X and Y coordinates */
2056         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
2057                                 CCP_ECC_MODULUS_BYTES);
2058         dst.address += CCP_ECC_OUTPUT_SIZE;
2059         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
2060                                 CCP_ECC_MODULUS_BYTES);
2061         dst.address += CCP_ECC_OUTPUT_SIZE;
2062
2063         /* Restore the workarea address */
2064         dst.address = save;
2065
2066 e_dst:
2067         ccp_dm_free(&dst);
2068
2069 e_src:
2070         ccp_dm_free(&src);
2071
2072         return ret;
2073 }
2074
2075 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2076 {
2077         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2078
2079         ecc->ecc_result = 0;
2080
2081         if (!ecc->mod ||
2082             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2083                 return -EINVAL;
2084
2085         switch (ecc->function) {
2086         case CCP_ECC_FUNCTION_MMUL_384BIT:
2087         case CCP_ECC_FUNCTION_MADD_384BIT:
2088         case CCP_ECC_FUNCTION_MINV_384BIT:
2089                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2090
2091         case CCP_ECC_FUNCTION_PADD_384BIT:
2092         case CCP_ECC_FUNCTION_PMUL_384BIT:
2093         case CCP_ECC_FUNCTION_PDBL_384BIT:
2094                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2095
2096         default:
2097                 return -EINVAL;
2098         }
2099 }
2100
2101 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2102 {
2103         int ret;
2104
2105         cmd->engine_error = 0;
2106         cmd_q->cmd_error = 0;
2107         cmd_q->int_rcvd = 0;
2108         cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
2109
2110         switch (cmd->engine) {
2111         case CCP_ENGINE_AES:
2112                 ret = ccp_run_aes_cmd(cmd_q, cmd);
2113                 break;
2114         case CCP_ENGINE_XTS_AES_128:
2115                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2116                 break;
2117         case CCP_ENGINE_SHA:
2118                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2119                 break;
2120         case CCP_ENGINE_RSA:
2121                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2122                 break;
2123         case CCP_ENGINE_PASSTHRU:
2124                 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2125                 break;
2126         case CCP_ENGINE_ECC:
2127                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2128                 break;
2129         default:
2130                 ret = -EINVAL;
2131         }
2132
2133         return ret;
2134 }