drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/pci.h>
  17 #include <linux/interrupt.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <linux/ccp.h>
  20
  21 #include "ccp-dev.h"
  22
  23 /* SHA initial context values */
  24 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  25         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  26         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  27         cpu_to_be32(SHA1_H4),
  28 };
  29
  30 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  31         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  32         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  33         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  34         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  35 };
  36
  37 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  38         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  39         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  40         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  41         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  42 };
  43
  44 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  45                                         ccp_gen_jobid(ccp) : 0)
  46
  47 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  48 {
  49         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  50 }
  51
  52 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  53 {
  54         if (wa->dma_count)
  55                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  56
  57         wa->dma_count = 0;
  58 }
  59
  60 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  61                                 struct scatterlist *sg, u64 len,
  62                                 enum dma_data_direction dma_dir)
  63 {
  64         memset(wa, 0, sizeof(*wa));
  65
  66         wa->sg = sg;
  67         if (!sg)
  68                 return 0;
  69
  70         wa->nents = sg_nents_for_len(sg, len);
  71         if (wa->nents < 0)
  72                 return wa->nents;
  73
  74         wa->bytes_left = len;
  75         wa->sg_used = 0;
  76
  77         if (len == 0)
  78                 return 0;
  79
  80         if (dma_dir == DMA_NONE)
  81                 return 0;
  82
  83         wa->dma_sg = sg;
  84         wa->dma_dev = dev;
  85         wa->dma_dir = dma_dir;
  86         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
  87         if (!wa->dma_count)
  88                 return -ENOMEM;
  89
  90         return 0;
  91 }
  92
  93 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
  94 {
  95         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
  96
  97         if (!wa->sg)
  98                 return;
  99
 100         wa->sg_used += nbytes;
 101         wa->bytes_left -= nbytes;
 102         if (wa->sg_used == wa->sg->length) {
 103                 wa->sg = sg_next(wa->sg);
 104                 wa->sg_used = 0;
 105         }
 106 }
 107
 108 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 109 {
 110         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 111                 if (wa->address)
 112                         dma_pool_free(wa->dma_pool, wa->address,
 113                                       wa->dma.address);
 114         } else {
 115                 if (wa->dma.address)
 116                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 117                                          wa->dma.dir);
 118                 kfree(wa->address);
 119         }
 120
 121         wa->address = NULL;
 122         wa->dma.address = 0;
 123 }
 124
 125 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 126                                 struct ccp_cmd_queue *cmd_q,
 127                                 unsigned int len,
 128                                 enum dma_data_direction dir)
 129 {
 130         memset(wa, 0, sizeof(*wa));
 131
 132         if (!len)
 133                 return 0;
 134
 135         wa->dev = cmd_q->ccp->dev;
 136         wa->length = len;
 137
 138         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 139                 wa->dma_pool = cmd_q->dma_pool;
 140
 141                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 142                                              &wa->dma.address);
 143                 if (!wa->address)
 144                         return -ENOMEM;
 145
 146                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 147
 148                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 149         } else {
 150                 wa->address = kzalloc(len, GFP_KERNEL);
 151                 if (!wa->address)
 152                         return -ENOMEM;
 153
 154                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 155                                                  dir);
 156                 if (!wa->dma.address)
 157                         return -ENOMEM;
 158
 159                 wa->dma.length = len;
 160         }
 161         wa->dma.dir = dir;
 162
 163         return 0;
 164 }
 165
 166 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 167                             struct scatterlist *sg, unsigned int sg_offset,
 168                             unsigned int len)
 169 {
 170         WARN_ON(!wa->address);
 171
 172         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 173                                  0);
 174 }
 175
 176 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 177                             struct scatterlist *sg, unsigned int sg_offset,
 178                             unsigned int len)
 179 {
 180         WARN_ON(!wa->address);
 181
 182         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 183                                  1);
 184 }
 185
 186 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 187                                    struct scatterlist *sg,
 188                                    unsigned int len, unsigned int se_len,
 189                                    bool sign_extend)
 190 {
 191         unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
 192         u8 buffer[CCP_REVERSE_BUF_SIZE];
 193
 194         if (WARN_ON(se_len > sizeof(buffer)))
 195                 return -EINVAL;
 196
 197         sg_offset = len;
 198         dm_offset = 0;
 199         nbytes = len;
 200         while (nbytes) {
 201                 sb_len = min_t(unsigned int, nbytes, se_len);
 202                 sg_offset -= sb_len;
 203
 204                 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0);
 205                 for (i = 0; i < sb_len; i++)
 206                         wa->address[dm_offset + i] = buffer[sb_len - i - 1];
 207
 208                 dm_offset += sb_len;
 209                 nbytes -= sb_len;
 210
 211                 if ((sb_len != se_len) && sign_extend) {
 212                         /* Must sign-extend to nearest sign-extend length */
 213                         if (wa->address[dm_offset - 1] & 0x80)
 214                                 memset(wa->address + dm_offset, 0xff,
 215                                        se_len - sb_len);
 216                 }
 217         }
 218
 219         return 0;
 220 }
 221
 222 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 223                                     struct scatterlist *sg,
 224                                     unsigned int len)
 225 {
 226         unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
 227         u8 buffer[CCP_REVERSE_BUF_SIZE];
 228
 229         sg_offset = 0;
 230         dm_offset = len;
 231         nbytes = len;
 232         while (nbytes) {
 233                 sb_len = min_t(unsigned int, nbytes, sizeof(buffer));
 234                 dm_offset -= sb_len;
 235
 236                 for (i = 0; i < sb_len; i++)
 237                         buffer[sb_len - i - 1] = wa->address[dm_offset + i];
 238                 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1);
 239
 240                 sg_offset += sb_len;
 241                 nbytes -= sb_len;
 242         }
 243 }
 244
 245 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 246 {
 247         ccp_dm_free(&data->dm_wa);
 248         ccp_sg_free(&data->sg_wa);
 249 }
 250
 251 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 252                          struct scatterlist *sg, u64 sg_len,
 253                          unsigned int dm_len,
 254                          enum dma_data_direction dir)
 255 {
 256         int ret;
 257
 258         memset(data, 0, sizeof(*data));
 259
 260         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 261                                    dir);
 262         if (ret)
 263                 goto e_err;
 264
 265         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 266         if (ret)
 267                 goto e_err;
 268
 269         return 0;
 270
 271 e_err:
 272         ccp_free_data(data, cmd_q);
 273
 274         return ret;
 275 }
 276
 277 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 278 {
 279         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 280         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 281         unsigned int buf_count, nbytes;
 282
 283         /* Clear the buffer if setting it */
 284         if (!from)
 285                 memset(dm_wa->address, 0, dm_wa->length);
 286
 287         if (!sg_wa->sg)
 288                 return 0;
 289
 290         /* Perform the copy operation
 291          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 292          *   an unsigned int
 293          */
 294         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 295         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 296                                  nbytes, from);
 297
 298         /* Update the structures and generate the count */
 299         buf_count = 0;
 300         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 301                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 302                              dm_wa->length - buf_count);
 303                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 304
 305                 buf_count += nbytes;
 306                 ccp_update_sg_workarea(sg_wa, nbytes);
 307         }
 308
 309         return buf_count;
 310 }
 311
 312 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 313 {
 314         return ccp_queue_buf(data, 0);
 315 }
 316
 317 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 318 {
 319         return ccp_queue_buf(data, 1);
 320 }
 321
 322 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 323                              struct ccp_op *op, unsigned int block_size,
 324                              bool blocksize_op)
 325 {
 326         unsigned int sg_src_len, sg_dst_len, op_len;
 327
 328         /* The CCP can only DMA from/to one address each per operation. This
 329          * requires that we find the smallest DMA area between the source
 330          * and destination. The resulting len values will always be <= UINT_MAX
 331          * because the dma length is an unsigned int.
 332          */
 333         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 334         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 335
 336         if (dst) {
 337                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 338                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 339                 op_len = min(sg_src_len, sg_dst_len);
 340         } else {
 341                 op_len = sg_src_len;
 342         }
 343
 344         /* The data operation length will be at least block_size in length
 345          * or the smaller of available sg room remaining for the source or
 346          * the destination
 347          */
 348         op_len = max(op_len, block_size);
 349
 350         /* Unless we have to buffer data, there's no reason to wait */
 351         op->soc = 0;
 352
 353         if (sg_src_len < block_size) {
 354                 /* Not enough data in the sg element, so it
 355                  * needs to be buffered into a blocksize chunk
 356                  */
 357                 int cp_len = ccp_fill_queue_buf(src);
 358
 359                 op->soc = 1;
 360                 op->src.u.dma.address = src->dm_wa.dma.address;
 361                 op->src.u.dma.offset = 0;
 362                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 363         } else {
 364                 /* Enough data in the sg element, but we need to
 365                  * adjust for any previously copied data
 366                  */
 367                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 368                 op->src.u.dma.offset = src->sg_wa.sg_used;
 369                 op->src.u.dma.length = op_len & ~(block_size - 1);
 370
 371                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 372         }
 373
 374         if (dst) {
 375                 if (sg_dst_len < block_size) {
 376                         /* Not enough room in the sg element or we're on the
 377                          * last piece of data (when using padding), so the
 378                          * output needs to be buffered into a blocksize chunk
 379                          */
 380                         op->soc = 1;
 381                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 382                         op->dst.u.dma.offset = 0;
 383                         op->dst.u.dma.length = op->src.u.dma.length;
 384                 } else {
 385                         /* Enough room in the sg element, but we need to
 386                          * adjust for any previously used area
 387                          */
 388                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 389                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 390                         op->dst.u.dma.length = op->src.u.dma.length;
 391                 }
 392         }
 393 }
 394
 395 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 396                              struct ccp_op *op)
 397 {
 398         op->init = 0;
 399
 400         if (dst) {
 401                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 402                         ccp_empty_queue_buf(dst);
 403                 else
 404                         ccp_update_sg_workarea(&dst->sg_wa,
 405                                                op->dst.u.dma.length);
 406         }
 407 }
 408
 409 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 410                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 411                                u32 byte_swap, bool from)
 412 {
 413         struct ccp_op op;
 414
 415         memset(&op, 0, sizeof(op));
 416
 417         op.cmd_q = cmd_q;
 418         op.jobid = jobid;
 419         op.eom = 1;
 420
 421         if (from) {
 422                 op.soc = 1;
 423                 op.src.type = CCP_MEMTYPE_SB;
 424                 op.src.u.sb = sb;
 425                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 426                 op.dst.u.dma.address = wa->dma.address;
 427                 op.dst.u.dma.length = wa->length;
 428         } else {
 429                 op.src.type = CCP_MEMTYPE_SYSTEM;
 430                 op.src.u.dma.address = wa->dma.address;
 431                 op.src.u.dma.length = wa->length;
 432                 op.dst.type = CCP_MEMTYPE_SB;
 433                 op.dst.u.sb = sb;
 434         }
 435
 436         op.u.passthru.byte_swap = byte_swap;
 437
 438         return cmd_q->ccp->vdata->perform->passthru(&op);
 439 }
 440
 441 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 442                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 443                           u32 byte_swap)
 444 {
 445         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 446 }
 447
 448 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 449                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 450                             u32 byte_swap)
 451 {
 452         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 453 }
 454
 455 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 456                                 struct ccp_cmd *cmd)
 457 {
 458         struct ccp_aes_engine *aes = &cmd->u.aes;
 459         struct ccp_dm_workarea key, ctx;
 460         struct ccp_data src;
 461         struct ccp_op op;
 462         unsigned int dm_offset;
 463         int ret;
 464
 465         if (!((aes->key_len == AES_KEYSIZE_128) ||
 466               (aes->key_len == AES_KEYSIZE_192) ||
 467               (aes->key_len == AES_KEYSIZE_256)))
 468                 return -EINVAL;
 469
 470         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 471                 return -EINVAL;
 472
 473         if (aes->iv_len != AES_BLOCK_SIZE)
 474                 return -EINVAL;
 475
 476         if (!aes->key || !aes->iv || !aes->src)
 477                 return -EINVAL;
 478
 479         if (aes->cmac_final) {
 480                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 481                         return -EINVAL;
 482
 483                 if (!aes->cmac_key)
 484                         return -EINVAL;
 485         }
 486
 487         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 488         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 489
 490         ret = -EIO;
 491         memset(&op, 0, sizeof(op));
 492         op.cmd_q = cmd_q;
 493         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 494         op.sb_key = cmd_q->sb_key;
 495         op.sb_ctx = cmd_q->sb_ctx;
 496         op.init = 1;
 497         op.u.aes.type = aes->type;
 498         op.u.aes.mode = aes->mode;
 499         op.u.aes.action = aes->action;
 500
 501         /* All supported key sizes fit in a single (32-byte) SB entry
 502          * and must be in little endian format. Use the 256-bit byte
 503          * swap passthru option to convert from big endian to little
 504          * endian.
 505          */
 506         ret = ccp_init_dm_workarea(&key, cmd_q,
 507                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 508                                    DMA_TO_DEVICE);
 509         if (ret)
 510                 return ret;
 511
 512         dm_offset = CCP_SB_BYTES - aes->key_len;
 513         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 514         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 515                              CCP_PASSTHRU_BYTESWAP_256BIT);
 516         if (ret) {
 517                 cmd->engine_error = cmd_q->cmd_error;
 518                 goto e_key;
 519         }
 520
 521         /* The AES context fits in a single (32-byte) SB entry and
 522          * must be in little endian format. Use the 256-bit byte swap
 523          * passthru option to convert from big endian to little endian.
 524          */
 525         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 526                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 527                                    DMA_BIDIRECTIONAL);
 528         if (ret)
 529                 goto e_key;
 530
 531         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 532         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 533         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 534                              CCP_PASSTHRU_BYTESWAP_256BIT);
 535         if (ret) {
 536                 cmd->engine_error = cmd_q->cmd_error;
 537                 goto e_ctx;
 538         }
 539
 540         /* Send data to the CCP AES engine */
 541         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 542                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 543         if (ret)
 544                 goto e_ctx;
 545
 546         while (src.sg_wa.bytes_left) {
 547                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 548                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 549                         op.eom = 1;
 550
 551                         /* Push the K1/K2 key to the CCP now */
 552                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 553                                                op.sb_ctx,
 554                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 555                         if (ret) {
 556                                 cmd->engine_error = cmd_q->cmd_error;
 557                                 goto e_src;
 558                         }
 559
 560                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 561                                         aes->cmac_key_len);
 562                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 563                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 564                         if (ret) {
 565                                 cmd->engine_error = cmd_q->cmd_error;
 566                                 goto e_src;
 567                         }
 568                 }
 569
 570                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 571                 if (ret) {
 572                         cmd->engine_error = cmd_q->cmd_error;
 573                         goto e_src;
 574                 }
 575
 576                 ccp_process_data(&src, NULL, &op);
 577         }
 578
 579         /* Retrieve the AES context - convert from LE to BE using
 580          * 32-byte (256-bit) byteswapping
 581          */
 582         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 583                                CCP_PASSTHRU_BYTESWAP_256BIT);
 584         if (ret) {
 585                 cmd->engine_error = cmd_q->cmd_error;
 586                 goto e_src;
 587         }
 588
 589         /* ...but we only need AES_BLOCK_SIZE bytes */
 590         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 591         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 592
 593 e_src:
 594         ccp_free_data(&src, cmd_q);
 595
 596 e_ctx:
 597         ccp_dm_free(&ctx);
 598
 599 e_key:
 600         ccp_dm_free(&key);
 601
 602         return ret;
 603 }
 604
 605 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 606 {
 607         struct ccp_aes_engine *aes = &cmd->u.aes;
 608         struct ccp_dm_workarea key, ctx;
 609         struct ccp_data src, dst;
 610         struct ccp_op op;
 611         unsigned int dm_offset;
 612         bool in_place = false;
 613         int ret;
 614
 615         if (aes->mode == CCP_AES_MODE_CMAC)
 616                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 617
 618         if (!((aes->key_len == AES_KEYSIZE_128) ||
 619               (aes->key_len == AES_KEYSIZE_192) ||
 620               (aes->key_len == AES_KEYSIZE_256)))
 621                 return -EINVAL;
 622
 623         if (((aes->mode == CCP_AES_MODE_ECB) ||
 624              (aes->mode == CCP_AES_MODE_CBC) ||
 625              (aes->mode == CCP_AES_MODE_CFB)) &&
 626             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 627                 return -EINVAL;
 628
 629         if (!aes->key || !aes->src || !aes->dst)
 630                 return -EINVAL;
 631
 632         if (aes->mode != CCP_AES_MODE_ECB) {
 633                 if (aes->iv_len != AES_BLOCK_SIZE)
 634                         return -EINVAL;
 635
 636                 if (!aes->iv)
 637                         return -EINVAL;
 638         }
 639
 640         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 641         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 642
 643         ret = -EIO;
 644         memset(&op, 0, sizeof(op));
 645         op.cmd_q = cmd_q;
 646         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 647         op.sb_key = cmd_q->sb_key;
 648         op.sb_ctx = cmd_q->sb_ctx;
 649         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 650         op.u.aes.type = aes->type;
 651         op.u.aes.mode = aes->mode;
 652         op.u.aes.action = aes->action;
 653
 654         /* All supported key sizes fit in a single (32-byte) SB entry
 655          * and must be in little endian format. Use the 256-bit byte
 656          * swap passthru option to convert from big endian to little
 657          * endian.
 658          */
 659         ret = ccp_init_dm_workarea(&key, cmd_q,
 660                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 661                                    DMA_TO_DEVICE);
 662         if (ret)
 663                 return ret;
 664
 665         dm_offset = CCP_SB_BYTES - aes->key_len;
 666         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 667         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 668                              CCP_PASSTHRU_BYTESWAP_256BIT);
 669         if (ret) {
 670                 cmd->engine_error = cmd_q->cmd_error;
 671                 goto e_key;
 672         }
 673
 674         /* The AES context fits in a single (32-byte) SB entry and
 675          * must be in little endian format. Use the 256-bit byte swap
 676          * passthru option to convert from big endian to little endian.
 677          */
 678         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 679                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 680                                    DMA_BIDIRECTIONAL);
 681         if (ret)
 682                 goto e_key;
 683
 684         if (aes->mode != CCP_AES_MODE_ECB) {
 685                 /* Load the AES context - convert to LE */
 686                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 687                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 688                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 689                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 690                 if (ret) {
 691                         cmd->engine_error = cmd_q->cmd_error;
 692                         goto e_ctx;
 693                 }
 694         }
 695
 696         /* Prepare the input and output data workareas. For in-place
 697          * operations we need to set the dma direction to BIDIRECTIONAL
 698          * and copy the src workarea to the dst workarea.
 699          */
 700         if (sg_virt(aes->src) == sg_virt(aes->dst))
 701                 in_place = true;
 702
 703         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 704                             AES_BLOCK_SIZE,
 705                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 706         if (ret)
 707                 goto e_ctx;
 708
 709         if (in_place) {
 710                 dst = src;
 711         } else {
 712                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
 713                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 714                 if (ret)
 715                         goto e_src;
 716         }
 717
 718         /* Send data to the CCP AES engine */
 719         while (src.sg_wa.bytes_left) {
 720                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 721                 if (!src.sg_wa.bytes_left) {
 722                         op.eom = 1;
 723
 724                         /* Since we don't retrieve the AES context in ECB
 725                          * mode we have to wait for the operation to complete
 726                          * on the last piece of data
 727                          */
 728                         if (aes->mode == CCP_AES_MODE_ECB)
 729                                 op.soc = 1;
 730                 }
 731
 732                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 733                 if (ret) {
 734                         cmd->engine_error = cmd_q->cmd_error;
 735                         goto e_dst;
 736                 }
 737
 738                 ccp_process_data(&src, &dst, &op);
 739         }
 740
 741         if (aes->mode != CCP_AES_MODE_ECB) {
 742                 /* Retrieve the AES context - convert from LE to BE using
 743                  * 32-byte (256-bit) byteswapping
 744                  */
 745                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 746                                        CCP_PASSTHRU_BYTESWAP_256BIT);
 747                 if (ret) {
 748                         cmd->engine_error = cmd_q->cmd_error;
 749                         goto e_dst;
 750                 }
 751
 752                 /* ...but we only need AES_BLOCK_SIZE bytes */
 753                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 754                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 755         }
 756
 757 e_dst:
 758         if (!in_place)
 759                 ccp_free_data(&dst, cmd_q);
 760
 761 e_src:
 762         ccp_free_data(&src, cmd_q);
 763
 764 e_ctx:
 765         ccp_dm_free(&ctx);
 766
 767 e_key:
 768         ccp_dm_free(&key);
 769
 770         return ret;
 771 }
 772
 773 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 774                                struct ccp_cmd *cmd)
 775 {
 776         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
 777         struct ccp_dm_workarea key, ctx;
 778         struct ccp_data src, dst;
 779         struct ccp_op op;
 780         unsigned int unit_size, dm_offset;
 781         bool in_place = false;
 782         int ret;
 783
 784         switch (xts->unit_size) {
 785         case CCP_XTS_AES_UNIT_SIZE_16:
 786                 unit_size = 16;
 787                 break;
 788         case CCP_XTS_AES_UNIT_SIZE_512:
 789                 unit_size = 512;
 790                 break;
 791         case CCP_XTS_AES_UNIT_SIZE_1024:
 792                 unit_size = 1024;
 793                 break;
 794         case CCP_XTS_AES_UNIT_SIZE_2048:
 795                 unit_size = 2048;
 796                 break;
 797         case CCP_XTS_AES_UNIT_SIZE_4096:
 798                 unit_size = 4096;
 799                 break;
 800
 801         default:
 802                 return -EINVAL;
 803         }
 804
 805         if (xts->key_len != AES_KEYSIZE_128)
 806                 return -EINVAL;
 807
 808         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
 809                 return -EINVAL;
 810
 811         if (xts->iv_len != AES_BLOCK_SIZE)
 812                 return -EINVAL;
 813
 814         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
 815                 return -EINVAL;
 816
 817         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
 818         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
 819
 820         ret = -EIO;
 821         memset(&op, 0, sizeof(op));
 822         op.cmd_q = cmd_q;
 823         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 824         op.sb_key = cmd_q->sb_key;
 825         op.sb_ctx = cmd_q->sb_ctx;
 826         op.init = 1;
 827         op.u.xts.action = xts->action;
 828         op.u.xts.unit_size = xts->unit_size;
 829
 830         /* All supported key sizes fit in a single (32-byte) SB entry
 831          * and must be in little endian format. Use the 256-bit byte
 832          * swap passthru option to convert from big endian to little
 833          * endian.
 834          */
 835         ret = ccp_init_dm_workarea(&key, cmd_q,
 836                                    CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 837                                    DMA_TO_DEVICE);
 838         if (ret)
 839                 return ret;
 840
 841         dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
 842         ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
 843         ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
 844         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 845                              CCP_PASSTHRU_BYTESWAP_256BIT);
 846         if (ret) {
 847                 cmd->engine_error = cmd_q->cmd_error;
 848                 goto e_key;
 849         }
 850
 851         /* The AES context fits in a single (32-byte) SB entry and
 852          * for XTS is already in little endian format so no byte swapping
 853          * is needed.
 854          */
 855         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 856                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 857                                    DMA_BIDIRECTIONAL);
 858         if (ret)
 859                 goto e_key;
 860
 861         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
 862         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 863                              CCP_PASSTHRU_BYTESWAP_NOOP);
 864         if (ret) {
 865                 cmd->engine_error = cmd_q->cmd_error;
 866                 goto e_ctx;
 867         }
 868
 869         /* Prepare the input and output data workareas. For in-place
 870          * operations we need to set the dma direction to BIDIRECTIONAL
 871          * and copy the src workarea to the dst workarea.
 872          */
 873         if (sg_virt(xts->src) == sg_virt(xts->dst))
 874                 in_place = true;
 875
 876         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
 877                             unit_size,
 878                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 879         if (ret)
 880                 goto e_ctx;
 881
 882         if (in_place) {
 883                 dst = src;
 884         } else {
 885                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
 886                                     unit_size, DMA_FROM_DEVICE);
 887                 if (ret)
 888                         goto e_src;
 889         }
 890
 891         /* Send data to the CCP AES engine */
 892         while (src.sg_wa.bytes_left) {
 893                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
 894                 if (!src.sg_wa.bytes_left)
 895                         op.eom = 1;
 896
 897                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
 898                 if (ret) {
 899                         cmd->engine_error = cmd_q->cmd_error;
 900                         goto e_dst;
 901                 }
 902
 903                 ccp_process_data(&src, &dst, &op);
 904         }
 905
 906         /* Retrieve the AES context - convert from LE to BE using
 907          * 32-byte (256-bit) byteswapping
 908          */
 909         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 910                                CCP_PASSTHRU_BYTESWAP_256BIT);
 911         if (ret) {
 912                 cmd->engine_error = cmd_q->cmd_error;
 913                 goto e_dst;
 914         }
 915
 916         /* ...but we only need AES_BLOCK_SIZE bytes */
 917         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 918         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
 919
 920 e_dst:
 921         if (!in_place)
 922                 ccp_free_data(&dst, cmd_q);
 923
 924 e_src:
 925         ccp_free_data(&src, cmd_q);
 926
 927 e_ctx:
 928         ccp_dm_free(&ctx);
 929
 930 e_key:
 931         ccp_dm_free(&key);
 932
 933         return ret;
 934 }
 935
 936 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 937 {
 938         struct ccp_sha_engine *sha = &cmd->u.sha;
 939         struct ccp_dm_workarea ctx;
 940         struct ccp_data src;
 941         struct ccp_op op;
 942         unsigned int ioffset, ooffset;
 943         unsigned int digest_size;
 944         int sb_count;
 945         const void *init;
 946         u64 block_size;
 947         int ctx_size;
 948         int ret;
 949
 950         switch (sha->type) {
 951         case CCP_SHA_TYPE_1:
 952                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
 953                         return -EINVAL;
 954                 block_size = SHA1_BLOCK_SIZE;
 955                 break;
 956         case CCP_SHA_TYPE_224:
 957                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
 958                         return -EINVAL;
 959                 block_size = SHA224_BLOCK_SIZE;
 960                 break;
 961         case CCP_SHA_TYPE_256:
 962                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
 963                         return -EINVAL;
 964                 block_size = SHA256_BLOCK_SIZE;
 965                 break;
 966         default:
 967                 return -EINVAL;
 968         }
 969
 970         if (!sha->ctx)
 971                 return -EINVAL;
 972
 973         if (!sha->final && (sha->src_len & (block_size - 1)))
 974                 return -EINVAL;
 975
 976         /* The version 3 device can't handle zero-length input */
 977         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
 978
 979                 if (!sha->src_len) {
 980                         unsigned int digest_len;
 981                         const u8 *sha_zero;
 982
 983                         /* Not final, just return */
 984                         if (!sha->final)
 985                                 return 0;
 986
 987                         /* CCP can't do a zero length sha operation so the
 988                          * caller must buffer the data.
 989                          */
 990                         if (sha->msg_bits)
 991                                 return -EINVAL;
 992
 993                         /* The CCP cannot perform zero-length sha operations
 994                          * so the caller is required to buffer data for the
 995                          * final operation. However, a sha operation for a
 996                          * message with a total length of zero is valid so
 997                          * known values are required to supply the result.
 998                          */
 999                         switch (sha->type) {
1000                         case CCP_SHA_TYPE_1:
1001                                 sha_zero = sha1_zero_message_hash;
1002                                 digest_len = SHA1_DIGEST_SIZE;
1003                                 break;
1004                         case CCP_SHA_TYPE_224:
1005                                 sha_zero = sha224_zero_message_hash;
1006                                 digest_len = SHA224_DIGEST_SIZE;
1007                                 break;
1008                         case CCP_SHA_TYPE_256:
1009                                 sha_zero = sha256_zero_message_hash;
1010                                 digest_len = SHA256_DIGEST_SIZE;
1011                                 break;
1012                         default:
1013                                 return -EINVAL;
1014                         }
1015
1016                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1017                                                  digest_len, 1);
1018
1019                         return 0;
1020                 }
1021         }
1022
1023         /* Set variables used throughout */
1024         switch (sha->type) {
1025         case CCP_SHA_TYPE_1:
1026                 digest_size = SHA1_DIGEST_SIZE;
1027                 init = (void *) ccp_sha1_init;
1028                 ctx_size = SHA1_DIGEST_SIZE;
1029                 sb_count = 1;
1030                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1031                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1032                 else
1033                         ooffset = ioffset = 0;
1034                 break;
1035         case CCP_SHA_TYPE_224:
1036                 digest_size = SHA224_DIGEST_SIZE;
1037                 init = (void *) ccp_sha224_init;
1038                 ctx_size = SHA256_DIGEST_SIZE;
1039                 sb_count = 1;
1040                 ioffset = 0;
1041                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1042                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1043                 else
1044                         ooffset = 0;
1045                 break;
1046         case CCP_SHA_TYPE_256:
1047                 digest_size = SHA256_DIGEST_SIZE;
1048                 init = (void *) ccp_sha256_init;
1049                 ctx_size = SHA256_DIGEST_SIZE;
1050                 sb_count = 1;
1051                 ooffset = ioffset = 0;
1052                 break;
1053         default:
1054                 ret = -EINVAL;
1055                 goto e_data;
1056         }
1057
1058         /* For zero-length plaintext the src pointer is ignored;
1059          * otherwise both parts must be valid
1060          */
1061         if (sha->src_len && !sha->src)
1062                 return -EINVAL;
1063
1064         memset(&op, 0, sizeof(op));
1065         op.cmd_q = cmd_q;
1066         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1067         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1068         op.u.sha.type = sha->type;
1069         op.u.sha.msg_bits = sha->msg_bits;
1070
1071         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1072                                    DMA_BIDIRECTIONAL);
1073         if (ret)
1074                 return ret;
1075         if (sha->first) {
1076                 switch (sha->type) {
1077                 case CCP_SHA_TYPE_1:
1078                 case CCP_SHA_TYPE_224:
1079                 case CCP_SHA_TYPE_256:
1080                         memcpy(ctx.address + ioffset, init, ctx_size);
1081                         break;
1082                 default:
1083                         ret = -EINVAL;
1084                         goto e_ctx;
1085                 }
1086         } else {
1087                 /* Restore the context */
1088                 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1089                                 sb_count * CCP_SB_BYTES);
1090         }
1091
1092         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1093                              CCP_PASSTHRU_BYTESWAP_256BIT);
1094         if (ret) {
1095                 cmd->engine_error = cmd_q->cmd_error;
1096                 goto e_ctx;
1097         }
1098
1099         if (sha->src) {
1100                 /* Send data to the CCP SHA engine; block_size is set above */
1101                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1102                                     block_size, DMA_TO_DEVICE);
1103                 if (ret)
1104                         goto e_ctx;
1105
1106                 while (src.sg_wa.bytes_left) {
1107                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1108                         if (sha->final && !src.sg_wa.bytes_left)
1109                                 op.eom = 1;
1110
1111                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1112                         if (ret) {
1113                                 cmd->engine_error = cmd_q->cmd_error;
1114                                 goto e_data;
1115                         }
1116
1117                         ccp_process_data(&src, NULL, &op);
1118                 }
1119         } else {
1120                 op.eom = 1;
1121                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1122                 if (ret) {
1123                         cmd->engine_error = cmd_q->cmd_error;
1124                         goto e_data;
1125                 }
1126         }
1127
1128         /* Retrieve the SHA context - convert from LE to BE using
1129          * 32-byte (256-bit) byteswapping to BE
1130          */
1131         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1132                                CCP_PASSTHRU_BYTESWAP_256BIT);
1133         if (ret) {
1134                 cmd->engine_error = cmd_q->cmd_error;
1135                 goto e_data;
1136         }
1137
1138         if (sha->final) {
1139                 /* Finishing up, so get the digest */
1140                 switch (sha->type) {
1141                 case CCP_SHA_TYPE_1:
1142                 case CCP_SHA_TYPE_224:
1143                 case CCP_SHA_TYPE_256:
1144                         ccp_get_dm_area(&ctx, ooffset,
1145                                         sha->ctx, 0,
1146                                         digest_size);
1147                         break;
1148                 default:
1149                         ret = -EINVAL;
1150                         goto e_ctx;
1151                 }
1152         } else {
1153                 /* Stash the context */
1154                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1155                                 sb_count * CCP_SB_BYTES);
1156         }
1157
1158         if (sha->final && sha->opad) {
1159                 /* HMAC operation, recursively perform final SHA */
1160                 struct ccp_cmd hmac_cmd;
1161                 struct scatterlist sg;
1162                 u8 *hmac_buf;
1163
1164                 if (sha->opad_len != block_size) {
1165                         ret = -EINVAL;
1166                         goto e_data;
1167                 }
1168
1169                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1170                 if (!hmac_buf) {
1171                         ret = -ENOMEM;
1172                         goto e_data;
1173                 }
1174                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1175
1176                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1177                 switch (sha->type) {
1178                 case CCP_SHA_TYPE_1:
1179                 case CCP_SHA_TYPE_224:
1180                 case CCP_SHA_TYPE_256:
1181                         memcpy(hmac_buf + block_size,
1182                                ctx.address + ooffset,
1183                                digest_size);
1184                         break;
1185                 default:
1186                         ret = -EINVAL;
1187                         goto e_ctx;
1188                 }
1189
1190                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1191                 hmac_cmd.engine = CCP_ENGINE_SHA;
1192                 hmac_cmd.u.sha.type = sha->type;
1193                 hmac_cmd.u.sha.ctx = sha->ctx;
1194                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1195                 hmac_cmd.u.sha.src = &sg;
1196                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1197                 hmac_cmd.u.sha.opad = NULL;
1198                 hmac_cmd.u.sha.opad_len = 0;
1199                 hmac_cmd.u.sha.first = 1;
1200                 hmac_cmd.u.sha.final = 1;
1201                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1202
1203                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1204                 if (ret)
1205                         cmd->engine_error = hmac_cmd.engine_error;
1206
1207                 kfree(hmac_buf);
1208         }
1209
1210 e_data:
1211         if (sha->src)
1212                 ccp_free_data(&src, cmd_q);
1213
1214 e_ctx:
1215         ccp_dm_free(&ctx);
1216
1217         return ret;
1218 }
1219
1220 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1221 {
1222         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1223         struct ccp_dm_workarea exp, src;
1224         struct ccp_data dst;
1225         struct ccp_op op;
1226         unsigned int sb_count, i_len, o_len;
1227         int ret;
1228
1229         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1230                 return -EINVAL;
1231
1232         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1233                 return -EINVAL;
1234
1235         /* The RSA modulus must precede the message being acted upon, so
1236          * it must be copied to a DMA area where the message and the
1237          * modulus can be concatenated.  Therefore the input buffer
1238          * length required is twice the output buffer length (which
1239          * must be a multiple of 256-bits).
1240          */
1241         o_len = ((rsa->key_size + 255) / 256) * 32;
1242         i_len = o_len * 2;
1243
1244         sb_count = o_len / CCP_SB_BYTES;
1245
1246         memset(&op, 0, sizeof(op));
1247         op.cmd_q = cmd_q;
1248         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1249         op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
1250
1251         if (!op.sb_key)
1252                 return -EIO;
1253
1254         /* The RSA exponent may span multiple (32-byte) SB entries and must
1255          * be in little endian format. Reverse copy each 32-byte chunk
1256          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1257          * and each byte within that chunk and do not perform any byte swap
1258          * operations on the passthru operation.
1259          */
1260         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1261         if (ret)
1262                 goto e_sb;
1263
1264         ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1265                                       CCP_SB_BYTES, false);
1266         if (ret)
1267                 goto e_exp;
1268         ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1269                              CCP_PASSTHRU_BYTESWAP_NOOP);
1270         if (ret) {
1271                 cmd->engine_error = cmd_q->cmd_error;
1272                 goto e_exp;
1273         }
1274
1275         /* Concatenate the modulus and the message. Both the modulus and
1276          * the operands must be in little endian format.  Since the input
1277          * is in big endian format it must be converted.
1278          */
1279         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1280         if (ret)
1281                 goto e_exp;
1282
1283         ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1284                                       CCP_SB_BYTES, false);
1285         if (ret)
1286                 goto e_src;
1287         src.address += o_len;   /* Adjust the address for the copy operation */
1288         ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1289                                       CCP_SB_BYTES, false);
1290         if (ret)
1291                 goto e_src;
1292         src.address -= o_len;   /* Reset the address to original value */
1293
1294         /* Prepare the output area for the operation */
1295         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1296                             o_len, DMA_FROM_DEVICE);
1297         if (ret)
1298                 goto e_src;
1299
1300         op.soc = 1;
1301         op.src.u.dma.address = src.dma.address;
1302         op.src.u.dma.offset = 0;
1303         op.src.u.dma.length = i_len;
1304         op.dst.u.dma.address = dst.dm_wa.dma.address;
1305         op.dst.u.dma.offset = 0;
1306         op.dst.u.dma.length = o_len;
1307
1308         op.u.rsa.mod_size = rsa->key_size;
1309         op.u.rsa.input_len = i_len;
1310
1311         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1312         if (ret) {
1313                 cmd->engine_error = cmd_q->cmd_error;
1314                 goto e_dst;
1315         }
1316
1317         ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1318
1319 e_dst:
1320         ccp_free_data(&dst, cmd_q);
1321
1322 e_src:
1323         ccp_dm_free(&src);
1324
1325 e_exp:
1326         ccp_dm_free(&exp);
1327
1328 e_sb:
1329         cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1330
1331         return ret;
1332 }
1333
1334 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1335                                 struct ccp_cmd *cmd)
1336 {
1337         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1338         struct ccp_dm_workarea mask;
1339         struct ccp_data src, dst;
1340         struct ccp_op op;
1341         bool in_place = false;
1342         unsigned int i;
1343         int ret = 0;
1344
1345         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1346                 return -EINVAL;
1347
1348         if (!pt->src || !pt->dst)
1349                 return -EINVAL;
1350
1351         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1352                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1353                         return -EINVAL;
1354                 if (!pt->mask)
1355                         return -EINVAL;
1356         }
1357
1358         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1359
1360         memset(&op, 0, sizeof(op));
1361         op.cmd_q = cmd_q;
1362         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1363
1364         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1365                 /* Load the mask */
1366                 op.sb_key = cmd_q->sb_key;
1367
1368                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1369                                            CCP_PASSTHRU_SB_COUNT *
1370                                            CCP_SB_BYTES,
1371                                            DMA_TO_DEVICE);
1372                 if (ret)
1373                         return ret;
1374
1375                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1376                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1377                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1378                 if (ret) {
1379                         cmd->engine_error = cmd_q->cmd_error;
1380                         goto e_mask;
1381                 }
1382         }
1383
1384         /* Prepare the input and output data workareas. For in-place
1385          * operations we need to set the dma direction to BIDIRECTIONAL
1386          * and copy the src workarea to the dst workarea.
1387          */
1388         if (sg_virt(pt->src) == sg_virt(pt->dst))
1389                 in_place = true;
1390
1391         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1392                             CCP_PASSTHRU_MASKSIZE,
1393                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1394         if (ret)
1395                 goto e_mask;
1396
1397         if (in_place) {
1398                 dst = src;
1399         } else {
1400                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1401                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1402                 if (ret)
1403                         goto e_src;
1404         }
1405
1406         /* Send data to the CCP Passthru engine
1407          *   Because the CCP engine works on a single source and destination
1408          *   dma address at a time, each entry in the source scatterlist
1409          *   (after the dma_map_sg call) must be less than or equal to the
1410          *   (remaining) length in the destination scatterlist entry and the
1411          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1412          */
1413         dst.sg_wa.sg_used = 0;
1414         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1415                 if (!dst.sg_wa.sg ||
1416                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1417                         ret = -EINVAL;
1418                         goto e_dst;
1419                 }
1420
1421                 if (i == src.sg_wa.dma_count) {
1422                         op.eom = 1;
1423                         op.soc = 1;
1424                 }
1425
1426                 op.src.type = CCP_MEMTYPE_SYSTEM;
1427                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1428                 op.src.u.dma.offset = 0;
1429                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1430
1431                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1432                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1433                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1434                 op.dst.u.dma.length = op.src.u.dma.length;
1435
1436                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1437                 if (ret) {
1438                         cmd->engine_error = cmd_q->cmd_error;
1439                         goto e_dst;
1440                 }
1441
1442                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1443                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1444                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1445                         dst.sg_wa.sg_used = 0;
1446                 }
1447                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1448         }
1449
1450 e_dst:
1451         if (!in_place)
1452                 ccp_free_data(&dst, cmd_q);
1453
1454 e_src:
1455         ccp_free_data(&src, cmd_q);
1456
1457 e_mask:
1458         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1459                 ccp_dm_free(&mask);
1460
1461         return ret;
1462 }
1463
1464 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
1465                                       struct ccp_cmd *cmd)
1466 {
1467         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
1468         struct ccp_dm_workarea mask;
1469         struct ccp_op op;
1470         int ret;
1471
1472         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1473                 return -EINVAL;
1474
1475         if (!pt->src_dma || !pt->dst_dma)
1476                 return -EINVAL;
1477
1478         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1479                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1480                         return -EINVAL;
1481                 if (!pt->mask)
1482                         return -EINVAL;
1483         }
1484
1485         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1486
1487         memset(&op, 0, sizeof(op));
1488         op.cmd_q = cmd_q;
1489         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1490
1491         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1492                 /* Load the mask */
1493                 op.sb_key = cmd_q->sb_key;
1494
1495                 mask.length = pt->mask_len;
1496                 mask.dma.address = pt->mask;
1497                 mask.dma.length = pt->mask_len;
1498
1499                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1500                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1501                 if (ret) {
1502                         cmd->engine_error = cmd_q->cmd_error;
1503                         return ret;
1504                 }
1505         }
1506
1507         /* Send data to the CCP Passthru engine */
1508         op.eom = 1;
1509         op.soc = 1;
1510
1511         op.src.type = CCP_MEMTYPE_SYSTEM;
1512         op.src.u.dma.address = pt->src_dma;
1513         op.src.u.dma.offset = 0;
1514         op.src.u.dma.length = pt->src_len;
1515
1516         op.dst.type = CCP_MEMTYPE_SYSTEM;
1517         op.dst.u.dma.address = pt->dst_dma;
1518         op.dst.u.dma.offset = 0;
1519         op.dst.u.dma.length = pt->src_len;
1520
1521         ret = cmd_q->ccp->vdata->perform->passthru(&op);
1522         if (ret)
1523                 cmd->engine_error = cmd_q->cmd_error;
1524
1525         return ret;
1526 }
1527
1528 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1529 {
1530         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1531         struct ccp_dm_workarea src, dst;
1532         struct ccp_op op;
1533         int ret;
1534         u8 *save;
1535
1536         if (!ecc->u.mm.operand_1 ||
1537             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1538                 return -EINVAL;
1539
1540         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1541                 if (!ecc->u.mm.operand_2 ||
1542                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1543                         return -EINVAL;
1544
1545         if (!ecc->u.mm.result ||
1546             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1547                 return -EINVAL;
1548
1549         memset(&op, 0, sizeof(op));
1550         op.cmd_q = cmd_q;
1551         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1552
1553         /* Concatenate the modulus and the operands. Both the modulus and
1554          * the operands must be in little endian format.  Since the input
1555          * is in big endian format it must be converted and placed in a
1556          * fixed length buffer.
1557          */
1558         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1559                                    DMA_TO_DEVICE);
1560         if (ret)
1561                 return ret;
1562
1563         /* Save the workarea address since it is updated in order to perform
1564          * the concatenation
1565          */
1566         save = src.address;
1567
1568         /* Copy the ECC modulus */
1569         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1570                                       CCP_ECC_OPERAND_SIZE, false);
1571         if (ret)
1572                 goto e_src;
1573         src.address += CCP_ECC_OPERAND_SIZE;
1574
1575         /* Copy the first operand */
1576         ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1577                                       ecc->u.mm.operand_1_len,
1578                                       CCP_ECC_OPERAND_SIZE, false);
1579         if (ret)
1580                 goto e_src;
1581         src.address += CCP_ECC_OPERAND_SIZE;
1582
1583         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1584                 /* Copy the second operand */
1585                 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1586                                               ecc->u.mm.operand_2_len,
1587                                               CCP_ECC_OPERAND_SIZE, false);
1588                 if (ret)
1589                         goto e_src;
1590                 src.address += CCP_ECC_OPERAND_SIZE;
1591         }
1592
1593         /* Restore the workarea address */
1594         src.address = save;
1595
1596         /* Prepare the output area for the operation */
1597         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1598                                    DMA_FROM_DEVICE);
1599         if (ret)
1600                 goto e_src;
1601
1602         op.soc = 1;
1603         op.src.u.dma.address = src.dma.address;
1604         op.src.u.dma.offset = 0;
1605         op.src.u.dma.length = src.length;
1606         op.dst.u.dma.address = dst.dma.address;
1607         op.dst.u.dma.offset = 0;
1608         op.dst.u.dma.length = dst.length;
1609
1610         op.u.ecc.function = cmd->u.ecc.function;
1611
1612         ret = cmd_q->ccp->vdata->perform->ecc(&op);
1613         if (ret) {
1614                 cmd->engine_error = cmd_q->cmd_error;
1615                 goto e_dst;
1616         }
1617
1618         ecc->ecc_result = le16_to_cpup(
1619                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1620         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1621                 ret = -EIO;
1622                 goto e_dst;
1623         }
1624
1625         /* Save the ECC result */
1626         ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1627
1628 e_dst:
1629         ccp_dm_free(&dst);
1630
1631 e_src:
1632         ccp_dm_free(&src);
1633
1634         return ret;
1635 }
1636
1637 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1638 {
1639         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1640         struct ccp_dm_workarea src, dst;
1641         struct ccp_op op;
1642         int ret;
1643         u8 *save;
1644
1645         if (!ecc->u.pm.point_1.x ||
1646             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1647             !ecc->u.pm.point_1.y ||
1648             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1649                 return -EINVAL;
1650
1651         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1652                 if (!ecc->u.pm.point_2.x ||
1653                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1654                     !ecc->u.pm.point_2.y ||
1655                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1656                         return -EINVAL;
1657         } else {
1658                 if (!ecc->u.pm.domain_a ||
1659                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1660                         return -EINVAL;
1661
1662                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1663                         if (!ecc->u.pm.scalar ||
1664                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1665                                 return -EINVAL;
1666         }
1667
1668         if (!ecc->u.pm.result.x ||
1669             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1670             !ecc->u.pm.result.y ||
1671             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1672                 return -EINVAL;
1673
1674         memset(&op, 0, sizeof(op));
1675         op.cmd_q = cmd_q;
1676         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1677
1678         /* Concatenate the modulus and the operands. Both the modulus and
1679          * the operands must be in little endian format.  Since the input
1680          * is in big endian format it must be converted and placed in a
1681          * fixed length buffer.
1682          */
1683         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1684                                    DMA_TO_DEVICE);
1685         if (ret)
1686                 return ret;
1687
1688         /* Save the workarea address since it is updated in order to perform
1689          * the concatenation
1690          */
1691         save = src.address;
1692
1693         /* Copy the ECC modulus */
1694         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1695                                       CCP_ECC_OPERAND_SIZE, false);
1696         if (ret)
1697                 goto e_src;
1698         src.address += CCP_ECC_OPERAND_SIZE;
1699
1700         /* Copy the first point X and Y coordinate */
1701         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1702                                       ecc->u.pm.point_1.x_len,
1703                                       CCP_ECC_OPERAND_SIZE, false);
1704         if (ret)
1705                 goto e_src;
1706         src.address += CCP_ECC_OPERAND_SIZE;
1707         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1708                                       ecc->u.pm.point_1.y_len,
1709                                       CCP_ECC_OPERAND_SIZE, false);
1710         if (ret)
1711                 goto e_src;
1712         src.address += CCP_ECC_OPERAND_SIZE;
1713
1714         /* Set the first point Z coordinate to 1 */
1715         *src.address = 0x01;
1716         src.address += CCP_ECC_OPERAND_SIZE;
1717
1718         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1719                 /* Copy the second point X and Y coordinate */
1720                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1721                                               ecc->u.pm.point_2.x_len,
1722                                               CCP_ECC_OPERAND_SIZE, false);
1723                 if (ret)
1724                         goto e_src;
1725                 src.address += CCP_ECC_OPERAND_SIZE;
1726                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1727                                               ecc->u.pm.point_2.y_len,
1728                                               CCP_ECC_OPERAND_SIZE, false);
1729                 if (ret)
1730                         goto e_src;
1731                 src.address += CCP_ECC_OPERAND_SIZE;
1732
1733                 /* Set the second point Z coordinate to 1 */
1734                 *src.address = 0x01;
1735                 src.address += CCP_ECC_OPERAND_SIZE;
1736         } else {
1737                 /* Copy the Domain "a" parameter */
1738                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1739                                               ecc->u.pm.domain_a_len,
1740                                               CCP_ECC_OPERAND_SIZE, false);
1741                 if (ret)
1742                         goto e_src;
1743                 src.address += CCP_ECC_OPERAND_SIZE;
1744
1745                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1746                         /* Copy the scalar value */
1747                         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1748                                                       ecc->u.pm.scalar_len,
1749                                                       CCP_ECC_OPERAND_SIZE,
1750                                                       false);
1751                         if (ret)
1752                                 goto e_src;
1753                         src.address += CCP_ECC_OPERAND_SIZE;
1754                 }
1755         }
1756
1757         /* Restore the workarea address */
1758         src.address = save;
1759
1760         /* Prepare the output area for the operation */
1761         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1762                                    DMA_FROM_DEVICE);
1763         if (ret)
1764                 goto e_src;
1765
1766         op.soc = 1;
1767         op.src.u.dma.address = src.dma.address;
1768         op.src.u.dma.offset = 0;
1769         op.src.u.dma.length = src.length;
1770         op.dst.u.dma.address = dst.dma.address;
1771         op.dst.u.dma.offset = 0;
1772         op.dst.u.dma.length = dst.length;
1773
1774         op.u.ecc.function = cmd->u.ecc.function;
1775
1776         ret = cmd_q->ccp->vdata->perform->ecc(&op);
1777         if (ret) {
1778                 cmd->engine_error = cmd_q->cmd_error;
1779                 goto e_dst;
1780         }
1781
1782         ecc->ecc_result = le16_to_cpup(
1783                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1784         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1785                 ret = -EIO;
1786                 goto e_dst;
1787         }
1788
1789         /* Save the workarea address since it is updated as we walk through
1790          * to copy the point math result
1791          */
1792         save = dst.address;
1793
1794         /* Save the ECC result X and Y coordinates */
1795         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1796                                 CCP_ECC_MODULUS_BYTES);
1797         dst.address += CCP_ECC_OUTPUT_SIZE;
1798         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1799                                 CCP_ECC_MODULUS_BYTES);
1800         dst.address += CCP_ECC_OUTPUT_SIZE;
1801
1802         /* Restore the workarea address */
1803         dst.address = save;
1804
1805 e_dst:
1806         ccp_dm_free(&dst);
1807
1808 e_src:
1809         ccp_dm_free(&src);
1810
1811         return ret;
1812 }
1813
1814 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1815 {
1816         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1817
1818         ecc->ecc_result = 0;
1819
1820         if (!ecc->mod ||
1821             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1822                 return -EINVAL;
1823
1824         switch (ecc->function) {
1825         case CCP_ECC_FUNCTION_MMUL_384BIT:
1826         case CCP_ECC_FUNCTION_MADD_384BIT:
1827         case CCP_ECC_FUNCTION_MINV_384BIT:
1828                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1829
1830         case CCP_ECC_FUNCTION_PADD_384BIT:
1831         case CCP_ECC_FUNCTION_PMUL_384BIT:
1832         case CCP_ECC_FUNCTION_PDBL_384BIT:
1833                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1834
1835         default:
1836                 return -EINVAL;
1837         }
1838 }
1839
1840 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1841 {
1842         int ret;
1843
1844         cmd->engine_error = 0;
1845         cmd_q->cmd_error = 0;
1846         cmd_q->int_rcvd = 0;
1847         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
1848
1849         switch (cmd->engine) {
1850         case CCP_ENGINE_AES:
1851                 ret = ccp_run_aes_cmd(cmd_q, cmd);
1852                 break;
1853         case CCP_ENGINE_XTS_AES_128:
1854                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
1855                 break;
1856         case CCP_ENGINE_SHA:
1857                 ret = ccp_run_sha_cmd(cmd_q, cmd);
1858                 break;
1859         case CCP_ENGINE_RSA:
1860                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
1861                 break;
1862         case CCP_ENGINE_PASSTHRU:
1863                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
1864                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
1865                 else
1866                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
1867                 break;
1868         case CCP_ENGINE_ECC:
1869                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
1870                 break;
1871         default:
1872                 ret = -EINVAL;
1873         }
1874
1875         return ret;
1876 }