omap_init_mcspi();
omap_init_sham();
omap_init_aes();
+ omap_init_rng();
} else {
/* These can be removed when bindings are done */
omap_init_wl12xx_of();
}
omap_init_sti();
- omap_init_rng();
omap_init_vout();
return 0;
obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o
obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
+obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o
# These modules require assembler to support AVX.
ifeq ($(avx_supported),yes)
crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o
sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o
+crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o
}
/* camellia sboxes */
-const u64 camellia_sp10011110[256] = {
+__visible const u64 camellia_sp10011110[256] = {
0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
0x9e00009e9e9e9e00ULL,
};
-const u64 camellia_sp22000222[256] = {
+__visible const u64 camellia_sp22000222[256] = {
0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
0x3d3d0000003d3d3dULL,
};
-const u64 camellia_sp03303033[256] = {
+__visible const u64 camellia_sp03303033[256] = {
0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
0x004f4f004f004f4fULL,
};
-const u64 camellia_sp00444404[256] = {
+__visible const u64 camellia_sp00444404[256] = {
0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
0x00009e9e9e9e009eULL,
};
-const u64 camellia_sp02220222[256] = {
+__visible const u64 camellia_sp02220222[256] = {
0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
0x003d3d3d003d3d3dULL,
};
-const u64 camellia_sp30333033[256] = {
+__visible const u64 camellia_sp30333033[256] = {
0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
0x4f004f4f4f004f4fULL,
};
-const u64 camellia_sp44044404[256] = {
+__visible const u64 camellia_sp44044404[256] = {
0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
0x9e9e009e9e9e009eULL,
};
-const u64 camellia_sp11101110[256] = {
+__visible const u64 camellia_sp11101110[256] = {
0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
subRL[1] ^= (subRL[1] & ~subRL[9]) << 32;
/* modified for FLinv(kl2) */
- dw = (subRL[1] & subRL[9]) >> 32,
- subRL[1] ^= rol32(dw, 1);
+ dw = (subRL[1] & subRL[9]) >> 32;
+ subRL[1] ^= rol32(dw, 1);
/* round 8 */
subRL[11] ^= subRL[1];
subRL[1] ^= (subRL[1] & ~subRL[17]) << 32;
/* modified for FLinv(kl4) */
- dw = (subRL[1] & subRL[17]) >> 32,
- subRL[1] ^= rol32(dw, 1);
+ dw = (subRL[1] & subRL[17]) >> 32;
+ subRL[1] ^= rol32(dw, 1);
/* round 14 */
subRL[19] ^= subRL[1];
} else {
subRL[1] ^= (subRL[1] & ~subRL[25]) << 32;
/* modified for FLinv(kl6) */
- dw = (subRL[1] & subRL[25]) >> 32,
- subRL[1] ^= rol32(dw, 1);
+ dw = (subRL[1] & subRL[25]) >> 32;
+ subRL[1] ^= rol32(dw, 1);
/* round 20 */
subRL[27] ^= subRL[1];
kw4 ^= (kw4 & ~subRL[24]) << 32;
/* modified for FL(kl5) */
- dw = (kw4 & subRL[24]) >> 32,
- kw4 ^= rol32(dw, 1);
+ dw = (kw4 & subRL[24]) >> 32;
+ kw4 ^= rol32(dw, 1);
}
/* round 17 */
kw4 ^= (kw4 & ~subRL[16]) << 32;
/* modified for FL(kl3) */
- dw = (kw4 & subRL[16]) >> 32,
- kw4 ^= rol32(dw, 1);
+ dw = (kw4 & subRL[16]) >> 32;
+ kw4 ^= rol32(dw, 1);
/* round 11 */
subRL[14] ^= kw4;
kw4 ^= (kw4 & ~subRL[8]) << 32;
/* modified for FL(kl1) */
- dw = (kw4 & subRL[8]) >> 32,
- kw4 ^= rol32(dw, 1);
+ dw = (kw4 & subRL[8]) >> 32;
+ kw4 ^= rol32(dw, 1);
/* round 5 */
subRL[6] ^= kw4;
SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]); /* round 5 */
tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]);
- dw = tl & (subRL[8] >> 32), /* FL(kl1) */
- tr = subRL[10] ^ rol32(dw, 1);
+ dw = tl & (subRL[8] >> 32); /* FL(kl1) */
+ tr = subRL[10] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(7, subRL[6] ^ tt); /* round 6 */
SET_SUBKEY_LR(9, subRL[9]); /* FLinv(kl2) */
tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]);
- dw = tl & (subRL[9] >> 32), /* FLinv(kl2) */
- tr = subRL[7] ^ rol32(dw, 1);
+ dw = tl & (subRL[9] >> 32); /* FLinv(kl2) */
+ tr = subRL[7] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(10, subRL[11] ^ tt); /* round 7 */
SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]); /* round 11 */
tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]);
- dw = tl & (subRL[16] >> 32), /* FL(kl3) */
- tr = subRL[18] ^ rol32(dw, 1);
+ dw = tl & (subRL[16] >> 32); /* FL(kl3) */
+ tr = subRL[18] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(15, subRL[14] ^ tt); /* round 12 */
SET_SUBKEY_LR(17, subRL[17]); /* FLinv(kl4) */
tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]);
- dw = tl & (subRL[17] >> 32), /* FLinv(kl4) */
- tr = subRL[15] ^ rol32(dw, 1);
+ dw = tl & (subRL[17] >> 32); /* FLinv(kl4) */
+ tr = subRL[15] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(18, subRL[19] ^ tt); /* round 13 */
SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]); /* kw3 */
} else {
tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]);
- dw = tl & (subRL[24] >> 32), /* FL(kl5) */
- tr = subRL[26] ^ rol32(dw, 1);
+ dw = tl & (subRL[24] >> 32); /* FL(kl5) */
+ tr = subRL[26] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(23, subRL[22] ^ tt); /* round 18 */
SET_SUBKEY_LR(25, subRL[25]); /* FLinv(kl6) */
tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]);
- dw = tl & (subRL[25] >> 32), /* FLinv(kl6) */
- tr = subRL[23] ^ rol32(dw, 1);
+ dw = tl & (subRL[25] >> 32); /* FLinv(kl6) */
+ tr = subRL[23] ^ rol32(dw, 1);
tt = (tr | ((u64)tl << 32));
SET_SUBKEY_LR(26, subRL[27] ^ tt); /* round 19 */
--- /dev/null
+########################################################################
+# Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
+#
+# Copyright (c) 2013, Intel Corporation
+#
+# Authors:
+# Erdinc Ozturk <erdinc.ozturk@intel.com>
+# Vinodh Gopal <vinodh.gopal@intel.com>
+# James Guilford <james.guilford@intel.com>
+# Tim Chen <tim.c.chen@linux.intel.com>
+#
+# This software is available to you under a choice of one of two
+# licenses. You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# OpenIB.org BSD license below:
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the
+# distribution.
+#
+# * Neither the name of the Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+#
+# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+########################################################################
+# Function API:
+# UINT16 crc_t10dif_pcl(
+# UINT16 init_crc, //initial CRC value, 16 bits
+# const unsigned char *buf, //buffer pointer to calculate CRC on
+# UINT64 len //buffer length in bytes (64-bit data)
+# );
+#
+# Reference paper titled "Fast CRC Computation for Generic
+# Polynomials Using PCLMULQDQ Instruction"
+# URL: http://www.intel.com/content/dam/www/public/us/en/documents
+# /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+#
+#
+
+#include <linux/linkage.h>
+
+.text
+
+#define arg1 %rdi
+#define arg2 %rsi
+#define arg3 %rdx
+
+#define arg1_low32 %edi
+
+ENTRY(crc_t10dif_pcl)
+.align 16
+
+ # adjust the 16-bit initial_crc value, scale it to 32 bits
+ shl $16, arg1_low32
+
+ # Allocate Stack Space
+ mov %rsp, %rcx
+ sub $16*2, %rsp
+ # align stack to 16 byte boundary
+ and $~(0x10 - 1), %rsp
+
+ # check if smaller than 256
+ cmp $256, arg3
+
+ # for sizes less than 128, we can't fold 64B at a time...
+ jl _less_than_128
+
+
+ # load the initial crc value
+ movd arg1_low32, %xmm10 # initial crc
+
+ # crc value does not need to be byte-reflected, but it needs
+ # to be moved to the high part of the register.
+ # because data will be byte-reflected and will align with
+ # initial crc at correct place.
+ pslldq $12, %xmm10
+
+ movdqa SHUF_MASK(%rip), %xmm11
+ # receive the initial 64B data, xor the initial crc value
+ movdqu 16*0(arg2), %xmm0
+ movdqu 16*1(arg2), %xmm1
+ movdqu 16*2(arg2), %xmm2
+ movdqu 16*3(arg2), %xmm3
+ movdqu 16*4(arg2), %xmm4
+ movdqu 16*5(arg2), %xmm5
+ movdqu 16*6(arg2), %xmm6
+ movdqu 16*7(arg2), %xmm7
+
+ pshufb %xmm11, %xmm0
+ # XOR the initial_crc value
+ pxor %xmm10, %xmm0
+ pshufb %xmm11, %xmm1
+ pshufb %xmm11, %xmm2
+ pshufb %xmm11, %xmm3
+ pshufb %xmm11, %xmm4
+ pshufb %xmm11, %xmm5
+ pshufb %xmm11, %xmm6
+ pshufb %xmm11, %xmm7
+
+ movdqa rk3(%rip), %xmm10 #xmm10 has rk3 and rk4
+ #imm value of pclmulqdq instruction
+ #will determine which constant to use
+
+ #################################################################
+ # we subtract 256 instead of 128 to save one instruction from the loop
+ sub $256, arg3
+
+ # at this section of the code, there is 64*x+y (0<=y<64) bytes of
+ # buffer. The _fold_64_B_loop will fold 64B at a time
+ # until we have 64+y Bytes of buffer
+
+
+ # fold 64B at a time. This section of the code folds 4 xmm
+ # registers in parallel
+_fold_64_B_loop:
+
+ # update the buffer pointer
+ add $128, arg2 # buf += 64#
+
+ movdqu 16*0(arg2), %xmm9
+ movdqu 16*1(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm0, %xmm8
+ movdqa %xmm1, %xmm13
+ pclmulqdq $0x0 , %xmm10, %xmm0
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0 , %xmm10, %xmm1
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm0
+ xorps %xmm8 , %xmm0
+ pxor %xmm12, %xmm1
+ xorps %xmm13, %xmm1
+
+ movdqu 16*2(arg2), %xmm9
+ movdqu 16*3(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm2, %xmm8
+ movdqa %xmm3, %xmm13
+ pclmulqdq $0x0, %xmm10, %xmm2
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0, %xmm10, %xmm3
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm2
+ xorps %xmm8 , %xmm2
+ pxor %xmm12, %xmm3
+ xorps %xmm13, %xmm3
+
+ movdqu 16*4(arg2), %xmm9
+ movdqu 16*5(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm4, %xmm8
+ movdqa %xmm5, %xmm13
+ pclmulqdq $0x0, %xmm10, %xmm4
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0, %xmm10, %xmm5
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm4
+ xorps %xmm8 , %xmm4
+ pxor %xmm12, %xmm5
+ xorps %xmm13, %xmm5
+
+ movdqu 16*6(arg2), %xmm9
+ movdqu 16*7(arg2), %xmm12
+ pshufb %xmm11, %xmm9
+ pshufb %xmm11, %xmm12
+ movdqa %xmm6 , %xmm8
+ movdqa %xmm7 , %xmm13
+ pclmulqdq $0x0 , %xmm10, %xmm6
+ pclmulqdq $0x11, %xmm10, %xmm8
+ pclmulqdq $0x0 , %xmm10, %xmm7
+ pclmulqdq $0x11, %xmm10, %xmm13
+ pxor %xmm9 , %xmm6
+ xorps %xmm8 , %xmm6
+ pxor %xmm12, %xmm7
+ xorps %xmm13, %xmm7
+
+ sub $128, arg3
+
+ # check if there is another 64B in the buffer to be able to fold
+ jge _fold_64_B_loop
+ ##################################################################
+
+
+ add $128, arg2
+ # at this point, the buffer pointer is pointing at the last y Bytes
+ # of the buffer the 64B of folded data is in 4 of the xmm
+ # registers: xmm0, xmm1, xmm2, xmm3
+
+
+ # fold the 8 xmm registers to 1 xmm register with different constants
+
+ movdqa rk9(%rip), %xmm10
+ movdqa %xmm0, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm0
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm0, %xmm7
+
+ movdqa rk11(%rip), %xmm10
+ movdqa %xmm1, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm1
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm1, %xmm7
+
+ movdqa rk13(%rip), %xmm10
+ movdqa %xmm2, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm2
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm2, %xmm7
+
+ movdqa rk15(%rip), %xmm10
+ movdqa %xmm3, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm3
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm3, %xmm7
+
+ movdqa rk17(%rip), %xmm10
+ movdqa %xmm4, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm4
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm4, %xmm7
+
+ movdqa rk19(%rip), %xmm10
+ movdqa %xmm5, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm5
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ xorps %xmm5, %xmm7
+
+ movdqa rk1(%rip), %xmm10 #xmm10 has rk1 and rk2
+ #imm value of pclmulqdq instruction
+ #will determine which constant to use
+ movdqa %xmm6, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm6
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm6, %xmm7
+
+
+ # instead of 64, we add 48 to the loop counter to save 1 instruction
+ # from the loop instead of a cmp instruction, we use the negative
+ # flag with the jl instruction
+ add $128-16, arg3
+ jl _final_reduction_for_128
+
+ # now we have 16+y bytes left to reduce. 16 Bytes is in register xmm7
+ # and the rest is in memory. We can fold 16 bytes at a time if y>=16
+ # continue folding 16B at a time
+
+_16B_reduction_loop:
+ movdqa %xmm7, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm7
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ movdqu (arg2), %xmm0
+ pshufb %xmm11, %xmm0
+ pxor %xmm0 , %xmm7
+ add $16, arg2
+ sub $16, arg3
+ # instead of a cmp instruction, we utilize the flags with the
+ # jge instruction equivalent of: cmp arg3, 16-16
+ # check if there is any more 16B in the buffer to be able to fold
+ jge _16B_reduction_loop
+
+ #now we have 16+z bytes left to reduce, where 0<= z < 16.
+ #first, we reduce the data in the xmm7 register
+
+
+_final_reduction_for_128:
+ # check if any more data to fold. If not, compute the CRC of
+ # the final 128 bits
+ add $16, arg3
+ je _128_done
+
+ # here we are getting data that is less than 16 bytes.
+ # since we know that there was data before the pointer, we can
+ # offset the input pointer before the actual point, to receive
+ # exactly 16 bytes. after that the registers need to be adjusted.
+_get_last_two_xmms:
+ movdqa %xmm7, %xmm2
+
+ movdqu -16(arg2, arg3), %xmm1
+ pshufb %xmm11, %xmm1
+
+ # get rid of the extra data that was loaded before
+ # load the shift constant
+ lea pshufb_shf_table+16(%rip), %rax
+ sub arg3, %rax
+ movdqu (%rax), %xmm0
+
+ # shift xmm2 to the left by arg3 bytes
+ pshufb %xmm0, %xmm2
+
+ # shift xmm7 to the right by 16-arg3 bytes
+ pxor mask1(%rip), %xmm0
+ pshufb %xmm0, %xmm7
+ pblendvb %xmm2, %xmm1 #xmm0 is implicit
+
+ # fold 16 Bytes
+ movdqa %xmm1, %xmm2
+ movdqa %xmm7, %xmm8
+ pclmulqdq $0x11, %xmm10, %xmm7
+ pclmulqdq $0x0 , %xmm10, %xmm8
+ pxor %xmm8, %xmm7
+ pxor %xmm2, %xmm7
+
+_128_done:
+ # compute crc of a 128-bit value
+ movdqa rk5(%rip), %xmm10 # rk5 and rk6 in xmm10
+ movdqa %xmm7, %xmm0
+
+ #64b fold
+ pclmulqdq $0x1, %xmm10, %xmm7
+ pslldq $8 , %xmm0
+ pxor %xmm0, %xmm7
+
+ #32b fold
+ movdqa %xmm7, %xmm0
+
+ pand mask2(%rip), %xmm0
+
+ psrldq $12, %xmm7
+ pclmulqdq $0x10, %xmm10, %xmm7
+ pxor %xmm0, %xmm7
+
+ #barrett reduction
+_barrett:
+ movdqa rk7(%rip), %xmm10 # rk7 and rk8 in xmm10
+ movdqa %xmm7, %xmm0
+ pclmulqdq $0x01, %xmm10, %xmm7
+ pslldq $4, %xmm7
+ pclmulqdq $0x11, %xmm10, %xmm7
+
+ pslldq $4, %xmm7
+ pxor %xmm0, %xmm7
+ pextrd $1, %xmm7, %eax
+
+_cleanup:
+ # scale the result back to 16 bits
+ shr $16, %eax
+ mov %rcx, %rsp
+ ret
+
+########################################################################
+
+.align 16
+_less_than_128:
+
+ # check if there is enough buffer to be able to fold 16B at a time
+ cmp $32, arg3
+ jl _less_than_32
+ movdqa SHUF_MASK(%rip), %xmm11
+
+ # now if there is, load the constants
+ movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10
+
+ movd arg1_low32, %xmm0 # get the initial crc value
+ pslldq $12, %xmm0 # align it to its correct place
+ movdqu (arg2), %xmm7 # load the plaintext
+ pshufb %xmm11, %xmm7 # byte-reflect the plaintext
+ pxor %xmm0, %xmm7
+
+
+ # update the buffer pointer
+ add $16, arg2
+
+ # update the counter. subtract 32 instead of 16 to save one
+ # instruction from the loop
+ sub $32, arg3
+
+ jmp _16B_reduction_loop
+
+
+.align 16
+_less_than_32:
+ # mov initial crc to the return value. this is necessary for
+ # zero-length buffers.
+ mov arg1_low32, %eax
+ test arg3, arg3
+ je _cleanup
+
+ movdqa SHUF_MASK(%rip), %xmm11
+
+ movd arg1_low32, %xmm0 # get the initial crc value
+ pslldq $12, %xmm0 # align it to its correct place
+
+ cmp $16, arg3
+ je _exact_16_left
+ jl _less_than_16_left
+
+ movdqu (arg2), %xmm7 # load the plaintext
+ pshufb %xmm11, %xmm7 # byte-reflect the plaintext
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+ add $16, arg2
+ sub $16, arg3
+ movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10
+ jmp _get_last_two_xmms
+
+
+.align 16
+_less_than_16_left:
+ # use stack space to load data less than 16 bytes, zero-out
+ # the 16B in memory first.
+
+ pxor %xmm1, %xmm1
+ mov %rsp, %r11
+ movdqa %xmm1, (%r11)
+
+ cmp $4, arg3
+ jl _only_less_than_4
+
+ # backup the counter value
+ mov arg3, %r9
+ cmp $8, arg3
+ jl _less_than_8_left
+
+ # load 8 Bytes
+ mov (arg2), %rax
+ mov %rax, (%r11)
+ add $8, %r11
+ sub $8, arg3
+ add $8, arg2
+_less_than_8_left:
+
+ cmp $4, arg3
+ jl _less_than_4_left
+
+ # load 4 Bytes
+ mov (arg2), %eax
+ mov %eax, (%r11)
+ add $4, %r11
+ sub $4, arg3
+ add $4, arg2
+_less_than_4_left:
+
+ cmp $2, arg3
+ jl _less_than_2_left
+
+ # load 2 Bytes
+ mov (arg2), %ax
+ mov %ax, (%r11)
+ add $2, %r11
+ sub $2, arg3
+ add $2, arg2
+_less_than_2_left:
+ cmp $1, arg3
+ jl _zero_left
+
+ # load 1 Byte
+ mov (arg2), %al
+ mov %al, (%r11)
+_zero_left:
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ # shl r9, 4
+ lea pshufb_shf_table+16(%rip), %rax
+ sub %r9, %rax
+ movdqu (%rax), %xmm0
+ pxor mask1(%rip), %xmm0
+
+ pshufb %xmm0, %xmm7
+ jmp _128_done
+
+.align 16
+_exact_16_left:
+ movdqu (arg2), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ jmp _128_done
+
+_only_less_than_4:
+ cmp $3, arg3
+ jl _only_less_than_3
+
+ # load 3 Bytes
+ mov (arg2), %al
+ mov %al, (%r11)
+
+ mov 1(arg2), %al
+ mov %al, 1(%r11)
+
+ mov 2(arg2), %al
+ mov %al, 2(%r11)
+
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ psrldq $5, %xmm7
+
+ jmp _barrett
+_only_less_than_3:
+ cmp $2, arg3
+ jl _only_less_than_2
+
+ # load 2 Bytes
+ mov (arg2), %al
+ mov %al, (%r11)
+
+ mov 1(arg2), %al
+ mov %al, 1(%r11)
+
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ psrldq $6, %xmm7
+
+ jmp _barrett
+_only_less_than_2:
+
+ # load 1 Byte
+ mov (arg2), %al
+ mov %al, (%r11)
+
+ movdqa (%rsp), %xmm7
+ pshufb %xmm11, %xmm7
+ pxor %xmm0 , %xmm7 # xor the initial crc value
+
+ psrldq $7, %xmm7
+
+ jmp _barrett
+
+ENDPROC(crc_t10dif_pcl)
+
+.data
+
+# precomputed constants
+# these constants are precomputed from the poly:
+# 0x8bb70000 (0x8bb7 scaled to 32 bits)
+.align 16
+# Q = 0x18BB70000
+# rk1 = 2^(32*3) mod Q << 32
+# rk2 = 2^(32*5) mod Q << 32
+# rk3 = 2^(32*15) mod Q << 32
+# rk4 = 2^(32*17) mod Q << 32
+# rk5 = 2^(32*3) mod Q << 32
+# rk6 = 2^(32*2) mod Q << 32
+# rk7 = floor(2^64/Q)
+# rk8 = Q
+rk1:
+.quad 0x2d56000000000000
+rk2:
+.quad 0x06df000000000000
+rk3:
+.quad 0x9d9d000000000000
+rk4:
+.quad 0x7cf5000000000000
+rk5:
+.quad 0x2d56000000000000
+rk6:
+.quad 0x1368000000000000
+rk7:
+.quad 0x00000001f65a57f8
+rk8:
+.quad 0x000000018bb70000
+
+rk9:
+.quad 0xceae000000000000
+rk10:
+.quad 0xbfd6000000000000
+rk11:
+.quad 0x1e16000000000000
+rk12:
+.quad 0x713c000000000000
+rk13:
+.quad 0xf7f9000000000000
+rk14:
+.quad 0x80a6000000000000
+rk15:
+.quad 0x044c000000000000
+rk16:
+.quad 0xe658000000000000
+rk17:
+.quad 0xad18000000000000
+rk18:
+.quad 0xa497000000000000
+rk19:
+.quad 0x6ee3000000000000
+rk20:
+.quad 0xe7b5000000000000
+
+
+
+mask1:
+.octa 0x80808080808080808080808080808080
+mask2:
+.octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF
+
+SHUF_MASK:
+.octa 0x000102030405060708090A0B0C0D0E0F
+
+pshufb_shf_table:
+# use these values for shift constants for the pshufb instruction
+# different alignments result in values as shown:
+# DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1
+# DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2
+# DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3
+# DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4
+# DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5
+# DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6
+# DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7
+# DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8
+# DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9
+# DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10
+# DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11
+# DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12
+# DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13
+# DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14
+# DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15
+.octa 0x8f8e8d8c8b8a89888786858483828100
+.octa 0x000e0d0c0b0a09080706050403020100
--- /dev/null
+/*
+ * Cryptographic API.
+ *
+ * T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions
+ *
+ * Copyright (C) 2013 Intel Corporation
+ * Author: Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/crc-t10dif.h>
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <asm/i387.h>
+#include <asm/cpufeature.h>
+#include <asm/cpu_device_id.h>
+
+asmlinkage __u16 crc_t10dif_pcl(__u16 crc, const unsigned char *buf,
+ size_t len);
+
+struct chksum_desc_ctx {
+ __u16 crc;
+};
+
+/*
+ * Steps through buffer one byte at at time, calculates reflected
+ * crc using table.
+ */
+
+static int chksum_init(struct shash_desc *desc)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ ctx->crc = 0;
+
+ return 0;
+}
+
+static int chksum_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ if (irq_fpu_usable()) {
+ kernel_fpu_begin();
+ ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
+ kernel_fpu_end();
+ } else
+ ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
+ return 0;
+}
+
+static int chksum_final(struct shash_desc *desc, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ *(__u16 *)out = ctx->crc;
+ return 0;
+}
+
+static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
+ u8 *out)
+{
+ if (irq_fpu_usable()) {
+ kernel_fpu_begin();
+ *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len);
+ kernel_fpu_end();
+ } else
+ *(__u16 *)out = crc_t10dif_generic(*crcp, data, len);
+ return 0;
+}
+
+static int chksum_finup(struct shash_desc *desc, const u8 *data,
+ unsigned int len, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, len, out);
+}
+
+static int chksum_digest(struct shash_desc *desc, const u8 *data,
+ unsigned int length, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, length, out);
+}
+
+static struct shash_alg alg = {
+ .digestsize = CRC_T10DIF_DIGEST_SIZE,
+ .init = chksum_init,
+ .update = chksum_update,
+ .final = chksum_final,
+ .finup = chksum_finup,
+ .digest = chksum_digest,
+ .descsize = sizeof(struct chksum_desc_ctx),
+ .base = {
+ .cra_name = "crct10dif",
+ .cra_driver_name = "crct10dif-pclmul",
+ .cra_priority = 200,
+ .cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static const struct x86_cpu_id crct10dif_cpu_id[] = {
+ X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id);
+
+static int __init crct10dif_intel_mod_init(void)
+{
+ if (!x86_match_cpu(crct10dif_cpu_id))
+ return -ENODEV;
+
+ return crypto_register_shash(&alg);
+}
+
+static void __exit crct10dif_intel_mod_fini(void)
+{
+ crypto_unregister_shash(&alg);
+}
+
+module_init(crct10dif_intel_mod_init);
+module_exit(crct10dif_intel_mod_fini);
+
+MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
+MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ.");
+MODULE_LICENSE("GPL");
+
+MODULE_ALIAS("crct10dif");
+MODULE_ALIAS("crct10dif-pclmul");
#define AVX_XOR_SPEED \
do { \
- if (cpu_has_avx) \
+ if (cpu_has_avx && cpu_has_osxsave) \
xor_speed(&xor_block_avx); \
} while (0)
#define AVX_SELECT(FASTEST) \
- (cpu_has_avx ? &xor_block_avx : FASTEST)
+ (cpu_has_avx && cpu_has_osxsave ? &xor_block_avx : FASTEST)
#else
which will enable any routine to use the CRC-32-IEEE 802.3 checksum
and gain better performance as compared with the table implementation.
+config CRYPTO_CRCT10DIF
+ tristate "CRCT10DIF algorithm"
+ select CRYPTO_HASH
+ help
+ CRC T10 Data Integrity Field computation is being cast as
+ a crypto transform. This allows for faster crc t10 diff
+ transforms to be used if they are available.
+
+config CRYPTO_CRCT10DIF_PCLMUL
+ tristate "CRCT10DIF PCLMULQDQ hardware acceleration"
+ depends on X86 && 64BIT && CRC_T10DIF
+ select CRYPTO_HASH
+ help
+ For x86_64 processors with SSE4.2 and PCLMULQDQ supported,
+ CRC T10 DIF PCLMULQDQ computation can be hardware
+ accelerated PCLMULQDQ instruction. This option will create
+ 'crct10dif-plcmul' module, which is faster when computing the
+ crct10dif checksum as compared with the generic table implementation.
+
config CRYPTO_GHASH
tristate "GHASH digest algorithm"
select CRYPTO_GF128MUL
obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
obj-$(CONFIG_CRYPTO_CRC32C) += crc32c.o
obj-$(CONFIG_CRYPTO_CRC32) += crc32.o
+obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif.o
obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
obj-$(CONFIG_CRYPTO_LZO) += lzo.o
obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
static const u32 rco_tab[10] = { 1, 2, 4, 8, 16, 32, 64, 128, 27, 54 };
-const u32 crypto_ft_tab[4][256] = {
+__visible const u32 crypto_ft_tab[4][256] = {
{
0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6,
0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591,
}
};
-const u32 crypto_fl_tab[4][256] = {
+__visible const u32 crypto_fl_tab[4][256] = {
{
0x00000063, 0x0000007c, 0x00000077, 0x0000007b,
0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5,
}
};
-const u32 crypto_it_tab[4][256] = {
+__visible const u32 crypto_it_tab[4][256] = {
{
0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a,
0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b,
}
};
-const u32 crypto_il_tab[4][256] = {
+__visible const u32 crypto_il_tab[4][256] = {
{
0x00000052, 0x00000009, 0x0000006a, 0x000000d5,
0x00000030, 0x00000036, 0x000000a5, 0x00000038,
/* round 6 */
subL[7] ^= subL[1]; subR[7] ^= subR[1];
subL[1] ^= subR[1] & ~subR[9];
- dw = subL[1] & subL[9],
- subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl2) */
+ dw = subL[1] & subL[9];
+ subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl2) */
/* round 8 */
subL[11] ^= subL[1]; subR[11] ^= subR[1];
/* round 10 */
/* round 12 */
subL[15] ^= subL[1]; subR[15] ^= subR[1];
subL[1] ^= subR[1] & ~subR[17];
- dw = subL[1] & subL[17],
- subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl4) */
+ dw = subL[1] & subL[17];
+ subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl4) */
/* round 14 */
subL[19] ^= subL[1]; subR[19] ^= subR[1];
/* round 16 */
kw4l = subL[25]; kw4r = subR[25];
} else {
subL[1] ^= subR[1] & ~subR[25];
- dw = subL[1] & subL[25],
- subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl6) */
+ dw = subL[1] & subL[25];
+ subR[1] ^= rol32(dw, 1); /* modified for FLinv(kl6) */
/* round 20 */
subL[27] ^= subL[1]; subR[27] ^= subR[1];
/* round 22 */
/* round 19 */
subL[26] ^= kw4l; subR[26] ^= kw4r;
kw4l ^= kw4r & ~subR[24];
- dw = kw4l & subL[24],
- kw4r ^= rol32(dw, 1); /* modified for FL(kl5) */
+ dw = kw4l & subL[24];
+ kw4r ^= rol32(dw, 1); /* modified for FL(kl5) */
}
/* round 17 */
subL[22] ^= kw4l; subR[22] ^= kw4r;
/* round 13 */
subL[18] ^= kw4l; subR[18] ^= kw4r;
kw4l ^= kw4r & ~subR[16];
- dw = kw4l & subL[16],
- kw4r ^= rol32(dw, 1); /* modified for FL(kl3) */
+ dw = kw4l & subL[16];
+ kw4r ^= rol32(dw, 1); /* modified for FL(kl3) */
/* round 11 */
subL[14] ^= kw4l; subR[14] ^= kw4r;
/* round 9 */
/* round 7 */
subL[10] ^= kw4l; subR[10] ^= kw4r;
kw4l ^= kw4r & ~subR[8];
- dw = kw4l & subL[8],
- kw4r ^= rol32(dw, 1); /* modified for FL(kl1) */
+ dw = kw4l & subL[8];
+ kw4r ^= rol32(dw, 1); /* modified for FL(kl1) */
/* round 5 */
subL[6] ^= kw4l; subR[6] ^= kw4r;
/* round 3 */
SUBKEY_L(6) = subL[5] ^ subL[7]; /* round 5 */
SUBKEY_R(6) = subR[5] ^ subR[7];
tl = subL[10] ^ (subR[10] & ~subR[8]);
- dw = tl & subL[8], /* FL(kl1) */
- tr = subR[10] ^ rol32(dw, 1);
+ dw = tl & subL[8]; /* FL(kl1) */
+ tr = subR[10] ^ rol32(dw, 1);
SUBKEY_L(7) = subL[6] ^ tl; /* round 6 */
SUBKEY_R(7) = subR[6] ^ tr;
SUBKEY_L(8) = subL[8]; /* FL(kl1) */
SUBKEY_L(9) = subL[9]; /* FLinv(kl2) */
SUBKEY_R(9) = subR[9];
tl = subL[7] ^ (subR[7] & ~subR[9]);
- dw = tl & subL[9], /* FLinv(kl2) */
- tr = subR[7] ^ rol32(dw, 1);
+ dw = tl & subL[9]; /* FLinv(kl2) */
+ tr = subR[7] ^ rol32(dw, 1);
SUBKEY_L(10) = tl ^ subL[11]; /* round 7 */
SUBKEY_R(10) = tr ^ subR[11];
SUBKEY_L(11) = subL[10] ^ subL[12]; /* round 8 */
SUBKEY_L(14) = subL[13] ^ subL[15]; /* round 11 */
SUBKEY_R(14) = subR[13] ^ subR[15];
tl = subL[18] ^ (subR[18] & ~subR[16]);
- dw = tl & subL[16], /* FL(kl3) */
- tr = subR[18] ^ rol32(dw, 1);
+ dw = tl & subL[16]; /* FL(kl3) */
+ tr = subR[18] ^ rol32(dw, 1);
SUBKEY_L(15) = subL[14] ^ tl; /* round 12 */
SUBKEY_R(15) = subR[14] ^ tr;
SUBKEY_L(16) = subL[16]; /* FL(kl3) */
SUBKEY_L(17) = subL[17]; /* FLinv(kl4) */
SUBKEY_R(17) = subR[17];
tl = subL[15] ^ (subR[15] & ~subR[17]);
- dw = tl & subL[17], /* FLinv(kl4) */
- tr = subR[15] ^ rol32(dw, 1);
+ dw = tl & subL[17]; /* FLinv(kl4) */
+ tr = subR[15] ^ rol32(dw, 1);
SUBKEY_L(18) = tl ^ subL[19]; /* round 13 */
SUBKEY_R(18) = tr ^ subR[19];
SUBKEY_L(19) = subL[18] ^ subL[20]; /* round 14 */
SUBKEY_R(24) = subR[24] ^ subR[23];
} else {
tl = subL[26] ^ (subR[26] & ~subR[24]);
- dw = tl & subL[24], /* FL(kl5) */
- tr = subR[26] ^ rol32(dw, 1);
+ dw = tl & subL[24]; /* FL(kl5) */
+ tr = subR[26] ^ rol32(dw, 1);
SUBKEY_L(23) = subL[22] ^ tl; /* round 18 */
SUBKEY_R(23) = subR[22] ^ tr;
SUBKEY_L(24) = subL[24]; /* FL(kl5) */
SUBKEY_L(25) = subL[25]; /* FLinv(kl6) */
SUBKEY_R(25) = subR[25];
tl = subL[23] ^ (subR[23] & ~subR[25]);
- dw = tl & subL[25], /* FLinv(kl6) */
- tr = subR[23] ^ rol32(dw, 1);
+ dw = tl & subL[25]; /* FLinv(kl6) */
+ tr = subR[23] ^ rol32(dw, 1);
SUBKEY_L(26) = tl ^ subL[27]; /* round 19 */
SUBKEY_R(26) = tr ^ subR[27];
SUBKEY_L(27) = subL[26] ^ subL[28]; /* round 20 */
#include <linux/module.h>
#include <crypto/cast_common.h>
-const u32 cast_s1[256] = {
+__visible const u32 cast_s1[256] = {
0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f,
0x9c004dd3, 0x6003e540, 0xcf9fc949,
0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0,
};
EXPORT_SYMBOL_GPL(cast_s1);
-const u32 cast_s2[256] = {
+__visible const u32 cast_s2[256] = {
0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a,
0xeec5207a, 0x55889c94, 0x72fc0651,
0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef,
};
EXPORT_SYMBOL_GPL(cast_s2);
-const u32 cast_s3[256] = {
+__visible const u32 cast_s3[256] = {
0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff,
0x369fe44b, 0x8c1fc644, 0xaececa90,
0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806,
};
EXPORT_SYMBOL_GPL(cast_s3);
-const u32 cast_s4[256] = {
+__visible const u32 cast_s4[256] = {
0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb,
0x64ad8c57, 0x85510443, 0xfa020ed1,
0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43,
--- /dev/null
+/*
+ * Cryptographic API.
+ *
+ * T10 Data Integrity Field CRC16 Crypto Transform
+ *
+ * Copyright (c) 2007 Oracle Corporation. All rights reserved.
+ * Written by Martin K. Petersen <martin.petersen@oracle.com>
+ * Copyright (C) 2013 Intel Corporation
+ * Author: Tim Chen <tim.c.chen@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/crc-t10dif.h>
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+
+struct chksum_desc_ctx {
+ __u16 crc;
+};
+
+/* Table generated using the following polynomium:
+ * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
+ * gt: 0x8bb7
+ */
+static const __u16 t10_dif_crc_table[256] = {
+ 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
+ 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
+ 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
+ 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
+ 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
+ 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
+ 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
+ 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
+ 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
+ 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
+ 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
+ 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
+ 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
+ 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
+ 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
+ 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
+ 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
+ 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
+ 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
+ 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
+ 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
+ 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
+ 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
+ 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
+ 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
+ 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
+ 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
+ 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
+ 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
+ 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
+ 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
+ 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
+};
+
+__u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, size_t len)
+{
+ unsigned int i;
+
+ for (i = 0 ; i < len ; i++)
+ crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff];
+
+ return crc;
+}
+EXPORT_SYMBOL(crc_t10dif_generic);
+
+/*
+ * Steps through buffer one byte at at time, calculates reflected
+ * crc using table.
+ */
+
+static int chksum_init(struct shash_desc *desc)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ ctx->crc = 0;
+
+ return 0;
+}
+
+static int chksum_update(struct shash_desc *desc, const u8 *data,
+ unsigned int length)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
+ return 0;
+}
+
+static int chksum_final(struct shash_desc *desc, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ *(__u16 *)out = ctx->crc;
+ return 0;
+}
+
+static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
+ u8 *out)
+{
+ *(__u16 *)out = crc_t10dif_generic(*crcp, data, len);
+ return 0;
+}
+
+static int chksum_finup(struct shash_desc *desc, const u8 *data,
+ unsigned int len, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, len, out);
+}
+
+static int chksum_digest(struct shash_desc *desc, const u8 *data,
+ unsigned int length, u8 *out)
+{
+ struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
+
+ return __chksum_finup(&ctx->crc, data, length, out);
+}
+
+static struct shash_alg alg = {
+ .digestsize = CRC_T10DIF_DIGEST_SIZE,
+ .init = chksum_init,
+ .update = chksum_update,
+ .final = chksum_final,
+ .finup = chksum_finup,
+ .digest = chksum_digest,
+ .descsize = sizeof(struct chksum_desc_ctx),
+ .base = {
+ .cra_name = "crct10dif",
+ .cra_driver_name = "crct10dif-generic",
+ .cra_priority = 100,
+ .cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static int __init crct10dif_mod_init(void)
+{
+ int ret;
+
+ ret = crypto_register_shash(&alg);
+ return ret;
+}
+
+static void __exit crct10dif_mod_fini(void)
+{
+ crypto_unregister_shash(&alg);
+}
+
+module_init(crct10dif_mod_init);
+module_exit(crct10dif_mod_fini);
+
+MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
+MODULE_DESCRIPTION("T10 DIF CRC calculation.");
+MODULE_LICENSE("GPL");
};
#undef Z
-#define Z(x) cpu_to_be32((x << 27) | (x >> 5))
+#define Z(x) cpu_to_be32(((x & 0x1f) << 27) | (x >> 5))
static const __be32 sbox1[256] = {
Z(0x77), Z(0x14), Z(0xa6), Z(0xfe), Z(0xb2), Z(0x5e), Z(0x8c), Z(0x3e),
Z(0x67), Z(0x6c), Z(0xa1), Z(0x0d), Z(0xc2), Z(0xa2), Z(0xc1), Z(0x85),
scatterwalk_done(&walk, out, 0);
}
EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy);
+
+int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes)
+{
+ int offset = 0, n = 0;
+
+ /* num_bytes is too small */
+ if (num_bytes < sg->length)
+ return -1;
+
+ do {
+ offset += sg->length;
+ n++;
+ sg = scatterwalk_sg_next(sg);
+
+ /* num_bytes is too large */
+ if (unlikely(!sg && (num_bytes < offset)))
+ return -1;
+ } while (sg && (num_bytes > offset));
+
+ return n;
+}
+EXPORT_SYMBOL_GPL(scatterwalk_bytes_sglen);
ret += tcrypt_test("ghash");
break;
+ case 47:
+ ret += tcrypt_test("crct10dif");
+ break;
+
case 100:
ret += tcrypt_test("hmac(md5)");
break;
test_hash_speed("crc32c", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
+ case 320:
+ test_hash_speed("crct10dif", sec, generic_hash_speed_template);
+ if (mode > 300 && mode < 400) break;
+
case 399:
break;
.count = CRC32C_TEST_VECTORS
}
}
+ }, {
+ .alg = "crct10dif",
+ .test = alg_test_hash,
+ .fips_allowed = 1,
+ .suite = {
+ .hash = {
+ .vecs = crct10dif_tv_template,
+ .count = CRCT10DIF_TEST_VECTORS
+ }
+ }
}, {
.alg = "cryptd(__driver-cbc-aes-aesni)",
.test = alg_test_null,
if (i >= 0)
rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
type, mask);
- if (j >= 0)
+ if (j >= 0 && j != i)
rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
type, mask);
}
};
+#define CRCT10DIF_TEST_VECTORS 3
+static struct hash_testvec crct10dif_tv_template[] = {
+ {
+ .plaintext = "abc",
+ .psize = 3,
+#ifdef __LITTLE_ENDIAN
+ .digest = "\x3b\x44",
+#else
+ .digest = "\x44\x3b",
+#endif
+ }, {
+ .plaintext = "1234567890123456789012345678901234567890"
+ "123456789012345678901234567890123456789",
+ .psize = 79,
+#ifdef __LITTLE_ENDIAN
+ .digest = "\x70\x4b",
+#else
+ .digest = "\x4b\x70",
+#endif
+ }, {
+ .plaintext =
+ "abcddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+ .psize = 56,
+#ifdef __LITTLE_ENDIAN
+ .digest = "\xe3\x9c",
+#else
+ .digest = "\x9c\xe3",
+#endif
+ .np = 2,
+ .tap = { 28, 28 }
+ }
+};
+
/*
* SHA1 test vectors from from FIPS PUB 180-1
* Long vector from CAVS 5.0
config HW_RANDOM_OMAP
tristate "OMAP Random Number Generator support"
- depends on HW_RANDOM && (ARCH_OMAP16XX || ARCH_OMAP2)
+ depends on HW_RANDOM && (ARCH_OMAP16XX || ARCH_OMAP2PLUS)
default HW_RANDOM
---help---
This driver provides kernel-side support for the Random Number
- Generator hardware found on OMAP16xx and OMAP24xx multimedia
- processors.
+ Generator hardware found on OMAP16xx, OMAP2/3/4/5 and AM33xx/AM43xx
+ multimedia processors.
To compile this driver as a module, choose M here: the
module will be called omap-rng.
goto out;
}
- clk_prepare_enable(mxc_rng->clk);
+ err = clk_prepare_enable(mxc_rng->clk);
+ if (err)
+ goto out;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
mxc_rng->mem = devm_ioremap_resource(&pdev->dev, res);
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/pm_runtime.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/interrupt.h>
#include <asm/io.h>
-#define RNG_OUT_REG 0x00 /* Output register */
-#define RNG_STAT_REG 0x04 /* Status register
- [0] = STAT_BUSY */
-#define RNG_ALARM_REG 0x24 /* Alarm register
- [7:0] = ALARM_COUNTER */
-#define RNG_CONFIG_REG 0x28 /* Configuration register
- [11:6] = RESET_COUNT
- [5:3] = RING2_DELAY
- [2:0] = RING1_DELAY */
-#define RNG_REV_REG 0x3c /* Revision register
- [7:0] = REV_NB */
-#define RNG_MASK_REG 0x40 /* Mask and reset register
- [2] = IT_EN
- [1] = SOFTRESET
- [0] = AUTOIDLE */
-#define RNG_SYSSTATUS 0x44 /* System status
- [0] = RESETDONE */
+#define RNG_REG_STATUS_RDY (1 << 0)
+
+#define RNG_REG_INTACK_RDY_MASK (1 << 0)
+#define RNG_REG_INTACK_SHUTDOWN_OFLO_MASK (1 << 1)
+#define RNG_SHUTDOWN_OFLO_MASK (1 << 1)
+
+#define RNG_CONTROL_STARTUP_CYCLES_SHIFT 16
+#define RNG_CONTROL_STARTUP_CYCLES_MASK (0xffff << 16)
+#define RNG_CONTROL_ENABLE_TRNG_SHIFT 10
+#define RNG_CONTROL_ENABLE_TRNG_MASK (1 << 10)
+
+#define RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT 16
+#define RNG_CONFIG_MAX_REFIL_CYCLES_MASK (0xffff << 16)
+#define RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT 0
+#define RNG_CONFIG_MIN_REFIL_CYCLES_MASK (0xff << 0)
+
+#define RNG_CONTROL_STARTUP_CYCLES 0xff
+#define RNG_CONFIG_MIN_REFIL_CYCLES 0x21
+#define RNG_CONFIG_MAX_REFIL_CYCLES 0x22
+
+#define RNG_ALARMCNT_ALARM_TH_SHIFT 0x0
+#define RNG_ALARMCNT_ALARM_TH_MASK (0xff << 0)
+#define RNG_ALARMCNT_SHUTDOWN_TH_SHIFT 16
+#define RNG_ALARMCNT_SHUTDOWN_TH_MASK (0x1f << 16)
+#define RNG_ALARM_THRESHOLD 0xff
+#define RNG_SHUTDOWN_THRESHOLD 0x4
+
+#define RNG_REG_FROENABLE_MASK 0xffffff
+#define RNG_REG_FRODETUNE_MASK 0xffffff
+
+#define OMAP2_RNG_OUTPUT_SIZE 0x4
+#define OMAP4_RNG_OUTPUT_SIZE 0x8
+
+enum {
+ RNG_OUTPUT_L_REG = 0,
+ RNG_OUTPUT_H_REG,
+ RNG_STATUS_REG,
+ RNG_INTMASK_REG,
+ RNG_INTACK_REG,
+ RNG_CONTROL_REG,
+ RNG_CONFIG_REG,
+ RNG_ALARMCNT_REG,
+ RNG_FROENABLE_REG,
+ RNG_FRODETUNE_REG,
+ RNG_ALARMMASK_REG,
+ RNG_ALARMSTOP_REG,
+ RNG_REV_REG,
+ RNG_SYSCONFIG_REG,
+};
+
+static const u16 reg_map_omap2[] = {
+ [RNG_OUTPUT_L_REG] = 0x0,
+ [RNG_STATUS_REG] = 0x4,
+ [RNG_CONFIG_REG] = 0x28,
+ [RNG_REV_REG] = 0x3c,
+ [RNG_SYSCONFIG_REG] = 0x40,
+};
+static const u16 reg_map_omap4[] = {
+ [RNG_OUTPUT_L_REG] = 0x0,
+ [RNG_OUTPUT_H_REG] = 0x4,
+ [RNG_STATUS_REG] = 0x8,
+ [RNG_INTMASK_REG] = 0xc,
+ [RNG_INTACK_REG] = 0x10,
+ [RNG_CONTROL_REG] = 0x14,
+ [RNG_CONFIG_REG] = 0x18,
+ [RNG_ALARMCNT_REG] = 0x1c,
+ [RNG_FROENABLE_REG] = 0x20,
+ [RNG_FRODETUNE_REG] = 0x24,
+ [RNG_ALARMMASK_REG] = 0x28,
+ [RNG_ALARMSTOP_REG] = 0x2c,
+ [RNG_REV_REG] = 0x1FE0,
+ [RNG_SYSCONFIG_REG] = 0x1FE4,
+};
+
+struct omap_rng_dev;
/**
- * struct omap_rng_private_data - RNG IP block-specific data
- * @base: virtual address of the beginning of the RNG IP block registers
- * @mem_res: struct resource * for the IP block registers physical memory
+ * struct omap_rng_pdata - RNG IP block-specific data
+ * @regs: Pointer to the register offsets structure.
+ * @data_size: No. of bytes in RNG output.
+ * @data_present: Callback to determine if data is available.
+ * @init: Callback for IP specific initialization sequence.
+ * @cleanup: Callback for IP specific cleanup sequence.
*/
-struct omap_rng_private_data {
- void __iomem *base;
- struct resource *mem_res;
+struct omap_rng_pdata {
+ u16 *regs;
+ u32 data_size;
+ u32 (*data_present)(struct omap_rng_dev *priv);
+ int (*init)(struct omap_rng_dev *priv);
+ void (*cleanup)(struct omap_rng_dev *priv);
};
-static inline u32 omap_rng_read_reg(struct omap_rng_private_data *priv, int reg)
+struct omap_rng_dev {
+ void __iomem *base;
+ struct device *dev;
+ const struct omap_rng_pdata *pdata;
+};
+
+static inline u32 omap_rng_read(struct omap_rng_dev *priv, u16 reg)
{
- return __raw_readl(priv->base + reg);
+ return __raw_readl(priv->base + priv->pdata->regs[reg]);
}
-static inline void omap_rng_write_reg(struct omap_rng_private_data *priv,
- int reg, u32 val)
+static inline void omap_rng_write(struct omap_rng_dev *priv, u16 reg,
+ u32 val)
{
- __raw_writel(val, priv->base + reg);
+ __raw_writel(val, priv->base + priv->pdata->regs[reg]);
}
static int omap_rng_data_present(struct hwrng *rng, int wait)
{
- struct omap_rng_private_data *priv;
+ struct omap_rng_dev *priv;
int data, i;
- priv = (struct omap_rng_private_data *)rng->priv;
+ priv = (struct omap_rng_dev *)rng->priv;
for (i = 0; i < 20; i++) {
- data = omap_rng_read_reg(priv, RNG_STAT_REG) ? 0 : 1;
+ data = priv->pdata->data_present(priv);
if (data || !wait)
break;
/* RNG produces data fast enough (2+ MBit/sec, even
static int omap_rng_data_read(struct hwrng *rng, u32 *data)
{
- struct omap_rng_private_data *priv;
+ struct omap_rng_dev *priv;
+ u32 data_size, i;
+
+ priv = (struct omap_rng_dev *)rng->priv;
+ data_size = priv->pdata->data_size;
+
+ for (i = 0; i < data_size / sizeof(u32); i++)
+ data[i] = omap_rng_read(priv, RNG_OUTPUT_L_REG + i);
+
+ if (priv->pdata->regs[RNG_INTACK_REG])
+ omap_rng_write(priv, RNG_INTACK_REG, RNG_REG_INTACK_RDY_MASK);
+ return data_size;
+}
+
+static int omap_rng_init(struct hwrng *rng)
+{
+ struct omap_rng_dev *priv;
- priv = (struct omap_rng_private_data *)rng->priv;
+ priv = (struct omap_rng_dev *)rng->priv;
+ return priv->pdata->init(priv);
+}
- *data = omap_rng_read_reg(priv, RNG_OUT_REG);
+static void omap_rng_cleanup(struct hwrng *rng)
+{
+ struct omap_rng_dev *priv;
- return sizeof(u32);
+ priv = (struct omap_rng_dev *)rng->priv;
+ priv->pdata->cleanup(priv);
}
static struct hwrng omap_rng_ops = {
.name = "omap",
.data_present = omap_rng_data_present,
.data_read = omap_rng_data_read,
+ .init = omap_rng_init,
+ .cleanup = omap_rng_cleanup,
+};
+
+static inline u32 omap2_rng_data_present(struct omap_rng_dev *priv)
+{
+ return omap_rng_read(priv, RNG_STATUS_REG) ? 0 : 1;
+}
+
+static int omap2_rng_init(struct omap_rng_dev *priv)
+{
+ omap_rng_write(priv, RNG_SYSCONFIG_REG, 0x1);
+ return 0;
+}
+
+static void omap2_rng_cleanup(struct omap_rng_dev *priv)
+{
+ omap_rng_write(priv, RNG_SYSCONFIG_REG, 0x0);
+}
+
+static struct omap_rng_pdata omap2_rng_pdata = {
+ .regs = (u16 *)reg_map_omap2,
+ .data_size = OMAP2_RNG_OUTPUT_SIZE,
+ .data_present = omap2_rng_data_present,
+ .init = omap2_rng_init,
+ .cleanup = omap2_rng_cleanup,
};
+#if defined(CONFIG_OF)
+static inline u32 omap4_rng_data_present(struct omap_rng_dev *priv)
+{
+ return omap_rng_read(priv, RNG_STATUS_REG) & RNG_REG_STATUS_RDY;
+}
+
+static int omap4_rng_init(struct omap_rng_dev *priv)
+{
+ u32 val;
+
+ /* Return if RNG is already running. */
+ if (omap_rng_read(priv, RNG_CONFIG_REG) & RNG_CONTROL_ENABLE_TRNG_MASK)
+ return 0;
+
+ val = RNG_CONFIG_MIN_REFIL_CYCLES << RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT;
+ val |= RNG_CONFIG_MAX_REFIL_CYCLES << RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT;
+ omap_rng_write(priv, RNG_CONFIG_REG, val);
+
+ omap_rng_write(priv, RNG_FRODETUNE_REG, 0x0);
+ omap_rng_write(priv, RNG_FROENABLE_REG, RNG_REG_FROENABLE_MASK);
+ val = RNG_ALARM_THRESHOLD << RNG_ALARMCNT_ALARM_TH_SHIFT;
+ val |= RNG_SHUTDOWN_THRESHOLD << RNG_ALARMCNT_SHUTDOWN_TH_SHIFT;
+ omap_rng_write(priv, RNG_ALARMCNT_REG, val);
+
+ val = RNG_CONTROL_STARTUP_CYCLES << RNG_CONTROL_STARTUP_CYCLES_SHIFT;
+ val |= RNG_CONTROL_ENABLE_TRNG_MASK;
+ omap_rng_write(priv, RNG_CONTROL_REG, val);
+
+ return 0;
+}
+
+static void omap4_rng_cleanup(struct omap_rng_dev *priv)
+{
+ int val;
+
+ val = omap_rng_read(priv, RNG_CONTROL_REG);
+ val &= ~RNG_CONTROL_ENABLE_TRNG_MASK;
+ omap_rng_write(priv, RNG_CONFIG_REG, val);
+}
+
+static irqreturn_t omap4_rng_irq(int irq, void *dev_id)
+{
+ struct omap_rng_dev *priv = dev_id;
+ u32 fro_detune, fro_enable;
+
+ /*
+ * Interrupt raised by a fro shutdown threshold, do the following:
+ * 1. Clear the alarm events.
+ * 2. De tune the FROs which are shutdown.
+ * 3. Re enable the shutdown FROs.
+ */
+ omap_rng_write(priv, RNG_ALARMMASK_REG, 0x0);
+ omap_rng_write(priv, RNG_ALARMSTOP_REG, 0x0);
+
+ fro_enable = omap_rng_read(priv, RNG_FROENABLE_REG);
+ fro_detune = ~fro_enable & RNG_REG_FRODETUNE_MASK;
+ fro_detune = fro_detune | omap_rng_read(priv, RNG_FRODETUNE_REG);
+ fro_enable = RNG_REG_FROENABLE_MASK;
+
+ omap_rng_write(priv, RNG_FRODETUNE_REG, fro_detune);
+ omap_rng_write(priv, RNG_FROENABLE_REG, fro_enable);
+
+ omap_rng_write(priv, RNG_INTACK_REG, RNG_REG_INTACK_SHUTDOWN_OFLO_MASK);
+
+ return IRQ_HANDLED;
+}
+
+static struct omap_rng_pdata omap4_rng_pdata = {
+ .regs = (u16 *)reg_map_omap4,
+ .data_size = OMAP4_RNG_OUTPUT_SIZE,
+ .data_present = omap4_rng_data_present,
+ .init = omap4_rng_init,
+ .cleanup = omap4_rng_cleanup,
+};
+
+static const struct of_device_id omap_rng_of_match[] = {
+ {
+ .compatible = "ti,omap2-rng",
+ .data = &omap2_rng_pdata,
+ },
+ {
+ .compatible = "ti,omap4-rng",
+ .data = &omap4_rng_pdata,
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, omap_rng_of_match);
+
+static int of_get_omap_rng_device_details(struct omap_rng_dev *priv,
+ struct platform_device *pdev)
+{
+ const struct of_device_id *match;
+ struct device *dev = &pdev->dev;
+ int irq, err;
+
+ match = of_match_device(of_match_ptr(omap_rng_of_match), dev);
+ if (!match) {
+ dev_err(dev, "no compatible OF match\n");
+ return -EINVAL;
+ }
+ priv->pdata = match->data;
+
+ if (of_device_is_compatible(dev->of_node, "ti,omap4-rng")) {
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ dev_err(dev, "%s: error getting IRQ resource - %d\n",
+ __func__, irq);
+ return irq;
+ }
+
+ err = devm_request_irq(dev, irq, omap4_rng_irq,
+ IRQF_TRIGGER_NONE, dev_name(dev), priv);
+ if (err) {
+ dev_err(dev, "unable to request irq %d, err = %d\n",
+ irq, err);
+ return err;
+ }
+ omap_rng_write(priv, RNG_INTMASK_REG, RNG_SHUTDOWN_OFLO_MASK);
+ }
+ return 0;
+}
+#else
+static int of_get_omap_rng_device_details(struct omap_rng_dev *omap_rng,
+ struct platform_device *pdev)
+{
+ return -EINVAL;
+}
+#endif
+
+static int get_omap_rng_device_details(struct omap_rng_dev *omap_rng)
+{
+ /* Only OMAP2/3 can be non-DT */
+ omap_rng->pdata = &omap2_rng_pdata;
+ return 0;
+}
+
static int omap_rng_probe(struct platform_device *pdev)
{
- struct omap_rng_private_data *priv;
+ struct omap_rng_dev *priv;
+ struct resource *res;
+ struct device *dev = &pdev->dev;
int ret;
- priv = kzalloc(sizeof(struct omap_rng_private_data), GFP_KERNEL);
+ priv = devm_kzalloc(dev, sizeof(struct omap_rng_dev), GFP_KERNEL);
if (!priv) {
dev_err(&pdev->dev, "could not allocate memory\n");
return -ENOMEM;
omap_rng_ops.priv = (unsigned long)priv;
platform_set_drvdata(pdev, priv);
+ priv->dev = dev;
- priv->mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- priv->base = devm_ioremap_resource(&pdev->dev, priv->mem_res);
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ priv->base = devm_ioremap_resource(dev, res);
if (IS_ERR(priv->base)) {
ret = PTR_ERR(priv->base);
goto err_ioremap;
}
- platform_set_drvdata(pdev, priv);
pm_runtime_enable(&pdev->dev);
pm_runtime_get_sync(&pdev->dev);
+ ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) :
+ get_omap_rng_device_details(priv);
+ if (ret)
+ goto err_ioremap;
+
ret = hwrng_register(&omap_rng_ops);
if (ret)
goto err_register;
dev_info(&pdev->dev, "OMAP Random Number Generator ver. %02x\n",
- omap_rng_read_reg(priv, RNG_REV_REG));
-
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x1);
+ omap_rng_read(priv, RNG_REV_REG));
return 0;
priv->base = NULL;
pm_runtime_disable(&pdev->dev);
err_ioremap:
- kfree(priv);
-
+ dev_err(dev, "initialization failed.\n");
return ret;
}
static int __exit omap_rng_remove(struct platform_device *pdev)
{
- struct omap_rng_private_data *priv = platform_get_drvdata(pdev);
+ struct omap_rng_dev *priv = platform_get_drvdata(pdev);
hwrng_unregister(&omap_rng_ops);
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x0);
+ priv->pdata->cleanup(priv);
pm_runtime_put_sync(&pdev->dev);
pm_runtime_disable(&pdev->dev);
- release_mem_region(priv->mem_res->start, resource_size(priv->mem_res));
-
- kfree(priv);
-
return 0;
}
static int omap_rng_suspend(struct device *dev)
{
- struct omap_rng_private_data *priv = dev_get_drvdata(dev);
+ struct omap_rng_dev *priv = dev_get_drvdata(dev);
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x0);
+ priv->pdata->cleanup(priv);
pm_runtime_put_sync(dev);
return 0;
static int omap_rng_resume(struct device *dev)
{
- struct omap_rng_private_data *priv = dev_get_drvdata(dev);
+ struct omap_rng_dev *priv = dev_get_drvdata(dev);
pm_runtime_get_sync(dev);
- omap_rng_write_reg(priv, RNG_MASK_REG, 0x1);
+ priv->pdata->init(priv);
return 0;
}
#endif
-/* work with hotplug and coldplug */
-MODULE_ALIAS("platform:omap_rng");
-
static struct platform_driver omap_rng_driver = {
.driver = {
.name = "omap_rng",
.owner = THIS_MODULE,
.pm = OMAP_RNG_PM,
+ .of_match_table = of_match_ptr(omap_rng_of_match),
},
.probe = omap_rng_probe,
.remove = __exit_p(omap_rng_remove),
};
-static int __init omap_rng_init(void)
-{
- return platform_driver_register(&omap_rng_driver);
-}
-
-static void __exit omap_rng_exit(void)
-{
- platform_driver_unregister(&omap_rng_driver);
-}
-
-module_init(omap_rng_init);
-module_exit(omap_rng_exit);
-
+module_platform_driver(omap_rng_driver);
+MODULE_ALIAS("platform:omap_rng");
MODULE_AUTHOR("Deepak Saxena (and others)");
MODULE_LICENSE("GPL");
static void __iomem *rng_base;
static struct clk *rng_clk;
-struct device *rng_dev;
+static struct device *rng_dev;
static inline u32 picoxcell_trng_read_csr(void)
{
struct resource *r;
int i;
- r = platform_get_resource(dev, IORESOURCE_MEM, 0);
- if (!r)
- return -EBUSY;
rngdev = devm_kzalloc(&dev->dev, sizeof(*rngdev), GFP_KERNEL);
if (!rngdev)
return -ENOMEM;
+ r = platform_get_resource(dev, IORESOURCE_MEM, 0);
rngdev->base = devm_ioremap_resource(&dev->dev, r);
if (IS_ERR(rngdev->base))
return PTR_ERR(rngdev->base);
#include <linux/kernel.h>
#include <linux/hw_random.h>
#include <linux/delay.h>
+#include <asm/cpu_device_id.h>
#include <asm/io.h>
#include <asm/msr.h>
#include <asm/cpufeature.h>
module_init(mod_init);
module_exit(mod_exit);
+static struct x86_cpu_id via_rng_cpu_id[] = {
+ X86_FEATURE_MATCH(X86_FEATURE_XSTORE),
+ {}
+};
+
MODULE_DESCRIPTION("H/W RNG driver for VIA CPU with PadLock");
MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(x86cpu, via_rng_cpu_id);
This option allows you to have support for AMCC crypto acceleration.
config CRYPTO_DEV_OMAP_SHAM
- tristate "Support for OMAP SHA1/MD5 hw accelerator"
- depends on ARCH_OMAP2 || ARCH_OMAP3
+ tristate "Support for OMAP MD5/SHA1/SHA2 hw accelerator"
+ depends on ARCH_OMAP2PLUS
select CRYPTO_SHA1
select CRYPTO_MD5
+ select CRYPTO_SHA256
+ select CRYPTO_SHA512
+ select CRYPTO_HMAC
help
- OMAP processors have SHA1/MD5 hw accelerator. Select this if you
- want to use the OMAP module for SHA1/MD5 algorithms.
+ OMAP processors have MD5/SHA1/SHA2 hw accelerator. Select this if you
+ want to use the OMAP module for MD5/SHA1/SHA2 algorithms.
config CRYPTO_DEV_OMAP_AES
tristate "Support for OMAP AES hw engine"
- depends on ARCH_OMAP2 || ARCH_OMAP3
+ depends on ARCH_OMAP2 || ARCH_OMAP3 || ARCH_OMAP2PLUS
select CRYPTO_AES
select CRYPTO_BLKCIPHER2
help
#include "crypto4xx_sa.h"
#include "crypto4xx_core.h"
-void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
- u32 save_iv, u32 ld_h, u32 ld_iv, u32 hdr_proc,
- u32 h, u32 c, u32 pad_type, u32 op_grp, u32 op,
- u32 dir)
+static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
+ u32 save_iv, u32 ld_h, u32 ld_iv,
+ u32 hdr_proc, u32 h, u32 c, u32 pad_type,
+ u32 op_grp, u32 op, u32 dir)
{
sa->sa_command_0.w = 0;
sa->sa_command_0.bf.save_hash_state = save_h;
sa->sa_command_0.bf.dir = dir;
}
-void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, u32 hmac_mc,
- u32 cfb, u32 esn, u32 sn_mask, u32 mute,
- u32 cp_pad, u32 cp_pay, u32 cp_hdr)
+static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
+ u32 hmac_mc, u32 cfb, u32 esn,
+ u32 sn_mask, u32 mute, u32 cp_pad,
+ u32 cp_pay, u32 cp_hdr)
{
sa->sa_command_1.w = 0;
sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
To compile this as a module, choose M here: the module
will be called caamrng.
+
+config CRYPTO_DEV_FSL_CAAM_DEBUG
+ bool "Enable debug output in CAAM driver"
+ depends on CRYPTO_DEV_FSL_CAAM
+ default n
+ help
+ Selecting this will enable printing of various debug
+ information in the CAAM driver.
#
# Makefile for the CAAM backend and dependent components
#
+ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y)
+ EXTRA_CFLAGS := -DDEBUG
+endif
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
#define CAAM_MAX_IV_LENGTH 16
/* length of descriptors text */
-#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
-
#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ)
#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ)
#ifdef DEBUG
/* for print_hex_dumps with line references */
-#define xstr(s) str(s)
-#define str(s) #s
#define debug(format, arg...) printk(format, arg)
#else
#define debug(format, arg...)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
keylen, enckeylen, authkeylen);
printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
ctx->split_key_len, ctx->split_key_pad_len);
- print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->split_key_pad_len + enckeylen, 1);
#endif
u32 *desc;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
aead_unmap(jrdev, edesc, req);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
req->assoclen , 1);
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
edesc->src_nents ? 100 : ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents ? 100 : req->cryptlen +
ctx->authsize + 4, 1);
offsetof(struct aead_edesc, hw_desc));
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
req->cryptlen, 1);
#endif
err = -EBADMSG;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4,
((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
sizeof(struct iphdr) + req->assoclen +
ctx->authsize + 36, 1);
if (!err && edesc->sec4_sg_bytes) {
struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
- print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
sg->length + ctx->authsize + 16, 1);
}
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->info,
edesc->src_nents > 1 ? 100 : ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
#endif
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->info,
ivsize, 1);
- print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
#endif
#ifdef DEBUG
debug("assoclen %d cryptlen %d authsize %d\n",
req->assoclen, req->cryptlen, authsize);
- print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
req->assoclen , 1);
- print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents ? 100 : ivsize, 1);
- print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents ? 100 : req->cryptlen, 1);
- print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
desc_bytes(sh_desc), 1);
#endif
#ifdef DEBUG
debug("assoclen %d cryptlen %d authsize %d\n",
req->assoclen, req->cryptlen, authsize);
- print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
req->assoclen , 1);
- print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
- print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
- print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
desc_bytes(sh_desc), 1);
#endif
int len, sec4_sg_index = 0;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->info,
ivsize, 1);
- print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
edesc->src_nents ? 100 : req->nbytes, 1);
#endif
init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
all_contig, true);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
return PTR_ERR(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
req->cryptlen, 1);
#endif
init_aead_job(ctx->sh_desc_dec,
ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
return PTR_ERR(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
req->cryptlen, 1);
#endif
init_aead_giv_job(ctx->sh_desc_givenc,
ctx->sh_desc_givenc_dma, edesc, req, contig);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
edesc->iv_dma = iv_dma;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
sec4_sg_bytes, 1);
#endif
init_ablkcipher_job(ctx->sh_desc_enc,
ctx->sh_desc_enc_dma, edesc, req, iv_contig);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
ctx->sh_desc_dec_dma, edesc, req, iv_contig);
desc = edesc->hw_desc;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
desc_bytes(edesc->hw_desc), 1);
#endif
#define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
/* length of descriptors text */
-#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
-
#define DESC_AHASH_BASE (4 * CAAM_CMD_SZ)
#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ)
#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
#ifdef DEBUG
/* for print_hex_dumps with line references */
-#define xstr(s) str(s)
-#define str(s) #s
#define debug(format, arg...) printk(format, arg)
#else
#define debug(format, arg...)
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash update shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ahash update shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash update first shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ahash update first shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash final shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash finup shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ahash digest shdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "ahash digest shdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "key_in@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
wait_for_completion_interruptible(&result.completion);
ret = result.err;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "digested key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR,
+ "digested key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in,
digestsize, 1);
#endif
#ifdef DEBUG
printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
ctx->split_key_len, ctx->split_key_pad_len);
- print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
#endif
return -ENOMEM;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->split_key_pad_len, 1);
#endif
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
kfree(edesc);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
if (req->result)
- print_hex_dump(KERN_ERR, "result@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
#endif
append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
*next_buflen = last_buflen;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
- print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
edesc->src_nents = 0;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
*next_buflen = 0;
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
- print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
digestsize);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc,
desc_bytes(desc), 1);
#endif
sg_copy(next_buf, req->src, req->nbytes);
}
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "next buf@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
*next_buflen, 1);
#endif
OP_ALG_RNG4_SK);
}
-struct instantiate_result {
- struct completion completion;
- int err;
-};
-
-static void rng4_init_done(struct device *dev, u32 *desc, u32 err,
- void *context)
-{
- struct instantiate_result *instantiation = context;
-
- if (err) {
- char tmp[CAAM_ERROR_STR_MAX];
-
- dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
- }
-
- instantiation->err = err;
- complete(&instantiation->completion);
-}
-
-static int instantiate_rng(struct device *jrdev)
+static int instantiate_rng(struct device *ctrldev)
{
- struct instantiate_result instantiation;
-
- dma_addr_t desc_dma;
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
+ struct caam_full __iomem *topregs;
+ unsigned int timeout = 100000;
u32 *desc;
- int ret;
+ int i, ret = 0;
desc = kmalloc(CAAM_CMD_SZ * 6, GFP_KERNEL | GFP_DMA);
if (!desc) {
- dev_err(jrdev, "cannot allocate RNG init descriptor memory\n");
+ dev_err(ctrldev, "can't allocate RNG init descriptor memory\n");
return -ENOMEM;
}
-
build_instantiation_desc(desc);
- desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE);
- init_completion(&instantiation.completion);
- ret = caam_jr_enqueue(jrdev, desc, rng4_init_done, &instantiation);
- if (!ret) {
- wait_for_completion_interruptible(&instantiation.completion);
- ret = instantiation.err;
- if (ret)
- dev_err(jrdev, "unable to instantiate RNG\n");
+
+ /* Set the bit to request direct access to DECO0 */
+ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl;
+ setbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE);
+
+ while (!(rd_reg32(&topregs->ctrl.deco_rq) & DECORR_DEN0) &&
+ --timeout)
+ cpu_relax();
+
+ if (!timeout) {
+ dev_err(ctrldev, "failed to acquire DECO 0\n");
+ ret = -EIO;
+ goto out;
}
- dma_unmap_single(jrdev, desc_dma, desc_bytes(desc), DMA_TO_DEVICE);
+ for (i = 0; i < desc_len(desc); i++)
+ topregs->deco.descbuf[i] = *(desc + i);
- kfree(desc);
+ wr_reg32(&topregs->deco.jr_ctl_hi, DECO_JQCR_WHL | DECO_JQCR_FOUR);
+ timeout = 10000000;
+ while ((rd_reg32(&topregs->deco.desc_dbg) & DECO_DBG_VALID) &&
+ --timeout)
+ cpu_relax();
+
+ if (!timeout) {
+ dev_err(ctrldev, "failed to instantiate RNG\n");
+ ret = -EIO;
+ }
+
+ clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE);
+out:
+ kfree(desc);
return ret;
}
if ((cha_vid & CHA_ID_RNG_MASK) >> CHA_ID_RNG_SHIFT >= 4 &&
!(rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & RDSTA_IF0)) {
kick_trng(pdev);
- ret = instantiate_rng(ctrlpriv->jrdev[0]);
+ ret = instantiate_rng(dev);
if (ret) {
caam_remove(pdev);
return ret;
/* NOTE: RTIC detection ought to go here, around Si time */
- /* Initialize queue allocator lock */
- spin_lock_init(&ctrlpriv->jr_alloc_lock);
-
caam_id = rd_reg64(&topregs->ctrl.perfmon.caam_id);
/* Report "alive" for developer to see */
#define CAAM_CMD_SZ sizeof(u32)
#define CAAM_PTR_SZ sizeof(dma_addr_t)
#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
+#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
#ifdef DEBUG
#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
#ifndef INTERN_H
#define INTERN_H
-#define JOBR_UNASSIGNED 0
-#define JOBR_ASSIGNED 1
-
/* Currently comes from Kconfig param as a ^2 (driver-required) */
#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE)
struct caam_job_ring __iomem *rregs; /* JobR's register space */
struct tasklet_struct irqtask;
int irq; /* One per queue */
- int assign; /* busy/free */
/* Job ring info */
int ringsize; /* Size of rings (assume input = output) */
struct device *dev;
struct device **jrdev; /* Alloc'ed array per sub-device */
- spinlock_t jr_alloc_lock;
struct platform_device *pdev;
/* Physical-presence section */
clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
}
-/**
- * caam_jr_register() - Alloc a ring for someone to use as needed. Returns
- * an ordinal of the rings allocated, else returns -ENODEV if no rings
- * are available.
- * @ctrldev: points to the controller level dev (parent) that
- * owns rings available for use.
- * @dev: points to where a pointer to the newly allocated queue's
- * dev can be written to if successful.
- **/
-int caam_jr_register(struct device *ctrldev, struct device **rdev)
-{
- struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
- struct caam_drv_private_jr *jrpriv = NULL;
- int ring;
-
- /* Lock, if free ring - assign, unlock */
- spin_lock(&ctrlpriv->jr_alloc_lock);
- for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) {
- jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]);
- if (jrpriv->assign == JOBR_UNASSIGNED) {
- jrpriv->assign = JOBR_ASSIGNED;
- *rdev = ctrlpriv->jrdev[ring];
- spin_unlock(&ctrlpriv->jr_alloc_lock);
- return ring;
- }
- }
-
- /* If assigned, write dev where caller needs it */
- spin_unlock(&ctrlpriv->jr_alloc_lock);
- *rdev = NULL;
-
- return -ENODEV;
-}
-EXPORT_SYMBOL(caam_jr_register);
-
-/**
- * caam_jr_deregister() - Deregister an API and release the queue.
- * Returns 0 if OK, -EBUSY if queue still contains pending entries
- * or unprocessed results at the time of the call
- * @dev - points to the dev that identifies the queue to
- * be released.
- **/
-int caam_jr_deregister(struct device *rdev)
-{
- struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
- struct caam_drv_private *ctrlpriv;
-
- /* Get the owning controller's private space */
- ctrlpriv = dev_get_drvdata(jrpriv->parentdev);
-
- /*
- * Make sure ring empty before release
- */
- if (rd_reg32(&jrpriv->rregs->outring_used) ||
- (rd_reg32(&jrpriv->rregs->inpring_avail) != JOBR_DEPTH))
- return -EBUSY;
-
- /* Release ring */
- spin_lock(&ctrlpriv->jr_alloc_lock);
- jrpriv->assign = JOBR_UNASSIGNED;
- spin_unlock(&ctrlpriv->jr_alloc_lock);
-
- return 0;
-}
-EXPORT_SYMBOL(caam_jr_deregister);
-
/**
* caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
* -EBUSY if the queue is full, -EIO if it cannot map the caller's
(JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
- jrp->assign = JOBR_UNASSIGNED;
return 0;
}
#define JR_H
/* Prototypes for backend-level services exposed to APIs */
-int caam_jr_register(struct device *ctrldev, struct device **rdev);
-int caam_jr_deregister(struct device *rdev);
int caam_jr_enqueue(struct device *dev, u32 *desc,
void (*cbk)(struct device *dev, u32 *desc, u32 status,
void *areq),
LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
- print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
wait_for_completion_interruptible(&result.completion);
ret = result.err;
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_out,
split_key_pad_len, 1);
#endif
#define MCFGR_DMA_RESET 0x10000000
#define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */
#define SCFGR_RDBENABLE 0x00000400
+#define DECORR_RQD0ENABLE 0x00000001 /* Enable DECO0 for direct access */
+#define DECORR_DEN0 0x00010000 /* DECO0 available for access*/
/* AXI read cache control */
#define MCFGR_ARCACHE_SHIFT 12
struct deco_sg_table sctr_tbl[4]; /* DxSTR - Scatter Tables */
u32 rsvd29[48];
u32 descbuf[64]; /* DxDESB - Descriptor buffer */
- u32 rsvd30[320];
+ u32 rscvd30[193];
+ u32 desc_dbg; /* DxDDR - DECO Debug Register */
+ u32 rsvd31[126];
};
+/* DECO DBG Register Valid Bit*/
+#define DECO_DBG_VALID 0x80000000
+#define DECO_JQCR_WHL 0x20000000
+#define DECO_JQCR_FOUR 0x10000000
+
/*
* Current top-level view of memory map is:
*
u64 rsvd[512];
struct caam_assurance assure;
struct caam_queue_if qi;
+ struct caam_deco deco;
};
#endif /* REGS_H */
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
- rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes,
- csbcpb->cpb.aes_cbc.iv);
- if (rc)
- goto out;
-
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
-
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
-
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ do {
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
+ processed, csbcpb->cpb.aes_cbc.iv);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_sg *nx_insg = nx_ctx->in_sg;
struct nx_sg *nx_outsg = nx_ctx->out_sg;
unsigned int iauth_len = 0;
- struct vio_pfo_op *op = NULL;
u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
int rc;
/* zero the ctr value */
memset(iv + 15 - iv[0], 0, iv[0] + 1);
+ /* page 78 of nx_wb.pdf has,
+ * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
+ * in length. If a full message is used, the AES CCA implementation
+ * restricts the maximum AAD length to 2^32 -1 bytes.
+ * If partial messages are used, the implementation supports
+ * 2^64 -1 bytes maximum AAD length.
+ *
+ * However, in the cryptoapi's aead_request structure,
+ * assoclen is an unsigned int, thus it cannot hold a length
+ * value greater than 2^32 - 1.
+ * Thus the AAD is further constrained by this and is never
+ * greater than 2^32.
+ */
+
if (!req->assoclen) {
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
} else if (req->assoclen <= 14) {
b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
b1 = nx_ctx->priv.ccm.iauth_tag;
iauth_len = req->assoclen;
+ } else if (req->assoclen <= 65280) {
+ /* if associated data is less than (2^16 - 2^8), we construct
+ * B1 differently and feed in the associated data to a CCA
+ * operation */
+ b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+ b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+ iauth_len = 14;
+ } else {
+ b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+ b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+ iauth_len = 10;
+ }
+
+ /* generate B0 */
+ rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
+ if (rc)
+ return rc;
+
+ /* generate B1:
+ * add control info for associated data
+ * RFC 3610 and NIST Special Publication 800-38C
+ */
+ if (b1) {
+ memset(b1, 0, 16);
+ if (req->assoclen <= 65280) {
+ *(u16 *)b1 = (u16)req->assoclen;
+ scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
+ iauth_len, SCATTERWALK_FROM_SG);
+ } else {
+ *(u16 *)b1 = (u16)(0xfffe);
+ *(u32 *)&b1[2] = (u32)req->assoclen;
+ scatterwalk_map_and_copy(b1 + 6, req->assoc, 0,
+ iauth_len, SCATTERWALK_FROM_SG);
+ }
+ }
+ /* now copy any remaining AAD to scatterlist and call nx... */
+ if (!req->assoclen) {
+ return rc;
+ } else if (req->assoclen <= 14) {
nx_insg = nx_build_sg_list(nx_insg, b1, 16, nx_ctx->ap->sglen);
nx_outsg = nx_build_sg_list(nx_outsg, tmp, 16,
nx_ctx->ap->sglen);
NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
- op = &nx_ctx->op;
result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
- } else if (req->assoclen <= 65280) {
- /* if associated data is less than (2^16 - 2^8), we construct
- * B1 differently and feed in the associated data to a CCA
- * operation */
- b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
- b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
- iauth_len = 14;
- /* remaining assoc data must have scatterlist built for it */
- nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen,
- req->assoc, iauth_len,
- req->assoclen - iauth_len);
- nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
- sizeof(struct nx_sg);
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
- op = &nx_ctx->op_aead;
- result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
} else {
- /* if associated data is less than (2^32), we construct B1
- * differently yet again and feed in the associated data to a
- * CCA operation */
- pr_err("associated data len is %u bytes (returning -EINVAL)\n",
- req->assoclen);
- rc = -EINVAL;
- }
+ u32 max_sg_len;
+ unsigned int processed = 0, to_process;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32,
+ nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ processed += iauth_len;
+
+ do {
+ to_process = min_t(u32, req->assoclen - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < req->assoclen) {
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
+ NX_FDM_INTERMEDIATE;
+ } else {
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
+ ~NX_FDM_INTERMEDIATE;
+ }
+
+ nx_insg = nx_walk_and_build(nx_ctx->in_sg,
+ nx_ctx->ap->sglen,
+ req->assoc, processed,
+ to_process);
+
+ nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
+ sizeof(struct nx_sg);
- rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
- if (rc)
- goto done;
+ result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
- if (b1) {
- memset(b1, 0, 16);
- *(u16 *)b1 = (u16)req->assoclen;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
- scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
- iauth_len, SCATTERWALK_FROM_SG);
+ memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
+ nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
+ AES_BLOCK_SIZE);
- rc = nx_hcall_sync(nx_ctx, op,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto done;
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen,
+ &(nx_ctx->stats->aes_bytes));
- memcpy(out, result, AES_BLOCK_SIZE);
+ processed += to_process;
+ } while (processed < req->assoclen);
+
+ result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
}
-done:
+
+ memcpy(out, result, AES_BLOCK_SIZE);
+
return rc;
}
unsigned int nbytes = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc = -1;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
nbytes -= authsize;
if (rc)
goto out;
- rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, nbytes,
- csbcpb->cpb.aes_ccm.iv_or_ctr);
- if (rc)
- goto out;
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
- NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
- NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ do {
+
+ /* to_process: the AES_BLOCK_SIZE data chunk to process in this
+ * update. This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
+ to_process, processed,
+ csbcpb->cpb.aes_ccm.iv_or_ctr);
+ if (rc)
+ goto out;
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ if (rc)
+ goto out;
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ /* for partial completion, copy following for next
+ * entry into loop...
+ */
+ memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+ csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_s0,
+ csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ /* update stats */
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
authsize) ? -EBADMSG : 0;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
unsigned int nbytes = req->cryptlen;
unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc = -1;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
csbcpb->cpb.aes_ccm.in_pat_or_b0);
if (rc)
goto out;
- rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, nbytes,
- csbcpb->cpb.aes_ccm.iv_or_ctr);
- if (rc)
- goto out;
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /* to process: the AES_BLOCK_SIZE data chunk to process in this
+ * update. This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
+ to_process, processed,
+ csbcpb->cpb.aes_ccm.iv_or_ctr);
+ if (rc)
+ goto out;
- NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
- NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ /* for partial completion, copy following for next
+ * entry into loop...
+ */
+ memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+ csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_s0,
+ csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ /* update stats */
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+
+ } while (processed < nbytes);
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
req->dst, nbytes, authsize,
SCATTERWALK_TO_SG);
+
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
- rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes,
- csbcpb->cpb.aes_ctr.iv);
- if (rc)
- goto out;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ do {
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
+ processed, csbcpb->cpb.aes_ctr.iv);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ processed += to_process;
+ } while (processed < nbytes);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
{
struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
int rc;
- if (nbytes > nx_ctx->ap->databytelen)
- return -EINVAL;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
if (enc)
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
else
NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
- rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes, NULL);
- if (rc)
- goto out;
+ do {
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, to_process,
+ processed, NULL);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ processed += to_process;
+ } while (processed < nbytes);
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct aead_request *req,
u8 *out)
{
+ int rc;
struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
- int rc = -EINVAL;
struct scatter_walk walk;
struct nx_sg *nx_sg = nx_ctx->in_sg;
+ unsigned int nbytes = req->assoclen;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
- if (req->assoclen > nx_ctx->ap->databytelen)
- goto out;
-
- if (req->assoclen <= AES_BLOCK_SIZE) {
+ if (nbytes <= AES_BLOCK_SIZE) {
scatterwalk_start(&walk, req->assoc);
- scatterwalk_copychunks(out, &walk, req->assoclen,
- SCATTERWALK_FROM_SG);
+ scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG);
scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0);
-
- rc = 0;
- goto out;
+ return 0;
}
- nx_sg = nx_walk_and_build(nx_sg, nx_ctx->ap->sglen, req->assoc, 0,
- req->assoclen);
- nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg) * sizeof(struct nx_sg);
+ NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_CONTINUATION;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_INTERMEDIATE;
+
+ nx_sg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen,
+ req->assoc, processed, to_process);
+ nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg)
+ * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ memcpy(csbcpb_aead->cpb.aes_gca.in_pat,
+ csbcpb_aead->cpb.aes_gca.out_pat,
+ AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+ memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
+
+ return rc;
+}
+
+static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
+{
+ int rc;
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *nx_sg;
+ unsigned int nbytes = req->assoclen;
+ unsigned int processed = 0, to_process;
+ u32 max_sg_len;
+
+ /* Set GMAC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GMAC;
+
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ /* Copy IV */
+ memcpy(csbcpb->cpb.aes_gcm.iv_or_cnt, desc->info, AES_BLOCK_SIZE);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ nx_sg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen,
+ req->assoc, processed, to_process);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - nx_sg)
+ * sizeof(struct nx_sg);
+
+ csbcpb->cpb.aes_gcm.bit_length_data = 0;
+ csbcpb->cpb.aes_gcm.bit_length_aad = 8 * nbytes;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+ csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_s0,
+ csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+out:
+ /* Restore GCM mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+ return rc;
+}
+
+static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc,
+ int enc)
+{
+ int rc;
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ char out[AES_BLOCK_SIZE];
+ struct nx_sg *in_sg, *out_sg;
+
+ /* For scenarios where the input message is zero length, AES CTR mode
+ * may be used. Set the source data to be a single block (16B) of all
+ * zeros, and set the input IV value to be the same as the GMAC IV
+ * value. - nx_wb 4.8.1.3 */
+
+ /* Change to ECB mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(csbcpb->cpb.aes_ecb.key, csbcpb->cpb.aes_gcm.key,
+ sizeof(csbcpb->cpb.aes_ecb.key));
+ if (enc)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ /* Encrypt the counter/IV */
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) desc->info,
+ AES_BLOCK_SIZE, nx_ctx->ap->sglen);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) out, sizeof(out),
+ nx_ctx->ap->sglen);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
if (rc)
goto out;
-
atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
- memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
+ /* Copy out the auth tag */
+ memcpy(csbcpb->cpb.aes_gcm.out_pat_or_mac, out,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)));
out:
+ /* Restore XCBC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+
+ /*
+ * ECB key uses the same region that GCM AAD and counter, so it's safe
+ * to just fill it with zeroes.
+ */
+ memset(csbcpb->cpb.aes_ecb.key, 0, sizeof(csbcpb->cpb.aes_ecb.key));
+
return rc;
}
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct blkcipher_desc desc;
unsigned int nbytes = req->cryptlen;
+ unsigned int processed = 0, to_process;
+ unsigned long irq_flags;
+ u32 max_sg_len;
int rc = -EINVAL;
- if (nbytes > nx_ctx->ap->databytelen)
- goto out;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
desc.info = nx_ctx->priv.gcm.iv;
/* initialize the counter */
*(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1;
- /* For scenarios where the input message is zero length, AES CTR mode
- * may be used. Set the source data to be a single block (16B) of all
- * zeros, and set the input IV value to be the same as the GMAC IV
- * value. - nx_wb 4.8.1.3 */
if (nbytes == 0) {
- char src[AES_BLOCK_SIZE] = {};
- struct scatterlist sg;
-
- desc.tfm = crypto_alloc_blkcipher("ctr(aes)", 0, 0);
- if (IS_ERR(desc.tfm)) {
- rc = -ENOMEM;
+ if (req->assoclen == 0)
+ rc = gcm_empty(req, &desc, enc);
+ else
+ rc = gmac(req, &desc);
+ if (rc)
goto out;
- }
-
- crypto_blkcipher_setkey(desc.tfm, csbcpb->cpb.aes_gcm.key,
- NX_CPB_KEY_SIZE(csbcpb) == NX_KS_AES_128 ? 16 :
- NX_CPB_KEY_SIZE(csbcpb) == NX_KS_AES_192 ? 24 : 32);
-
- sg_init_one(&sg, src, AES_BLOCK_SIZE);
- if (enc)
- crypto_blkcipher_encrypt_iv(&desc, req->dst, &sg,
- AES_BLOCK_SIZE);
else
- crypto_blkcipher_decrypt_iv(&desc, req->dst, &sg,
- AES_BLOCK_SIZE);
- crypto_free_blkcipher(desc.tfm);
-
- rc = 0;
- goto out;
+ goto mac;
}
- desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
-
+ /* Process associated data */
csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8;
-
if (req->assoclen) {
rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad);
if (rc)
goto out;
}
- if (enc)
+ /* Set flags for encryption */
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+ if (enc) {
NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
- else
+ } else {
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
nbytes -= crypto_aead_authsize(crypto_aead_reqtfm(req));
+ }
- csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
- rc = nx_build_sg_lists(nx_ctx, &desc, req->dst, req->src, nbytes,
- csbcpb->cpb.aes_gcm.iv_or_cnt);
- if (rc)
- goto out;
+ csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
+ desc.tfm = (struct crypto_blkcipher *) req->base.tfm;
+ rc = nx_build_sg_lists(nx_ctx, &desc, req->dst,
+ req->src, to_process, processed,
+ csbcpb->cpb.aes_gcm.iv_or_cnt);
+ if (rc)
+ goto out;
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- atomic_inc(&(nx_ctx->stats->aes_ops));
- atomic64_add(csbcpb->csb.processed_byte_count,
- &(nx_ctx->stats->aes_bytes));
+ memcpy(desc.info, csbcpb->cpb.aes_gcm.out_cnt, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+ csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_s0,
+ csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+mac:
if (enc) {
/* copy out the auth tag */
scatterwalk_map_and_copy(csbcpb->cpb.aes_gcm.out_pat_or_mac,
req->dst, nbytes,
crypto_aead_authsize(crypto_aead_reqtfm(req)),
SCATTERWALK_TO_SG);
- } else if (req->assoclen) {
+ } else {
u8 *itag = nx_ctx->priv.gcm.iauth_tag;
u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac;
- scatterwalk_map_and_copy(itag, req->dst, nbytes,
+ scatterwalk_map_and_copy(itag, req->src, nbytes,
crypto_aead_authsize(crypto_aead_reqtfm(req)),
SCATTERWALK_FROM_SG);
rc = memcmp(itag, otag,
-EBADMSG : 0;
}
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
return 0;
}
+/*
+ * Based on RFC 3566, for a zero-length message:
+ *
+ * n = 1
+ * K1 = E(K, 0x01010101010101010101010101010101)
+ * K3 = E(K, 0x03030303030303030303030303030303)
+ * E[0] = 0x00000000000000000000000000000000
+ * M[1] = 0x80000000000000000000000000000000 (0 length message with padding)
+ * E[1] = (K1, M[1] ^ E[0] ^ K3)
+ * Tag = M[1]
+ */
+static int nx_xcbc_empty(struct shash_desc *desc, u8 *out)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *in_sg, *out_sg;
+ u8 keys[2][AES_BLOCK_SIZE];
+ u8 key[32];
+ int rc = 0;
+
+ /* Change to ECB mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(key, csbcpb->cpb.aes_xcbc.key, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ecb.key, key, AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+ /* K1 and K3 base patterns */
+ memset(keys[0], 0x01, sizeof(keys[0]));
+ memset(keys[1], 0x03, sizeof(keys[1]));
+
+ /* Generate K1 and K3 encrypting the patterns */
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys, sizeof(keys),
+ nx_ctx->ap->sglen);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) keys, sizeof(keys),
+ nx_ctx->ap->sglen);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+ /* XOr K3 with the padding for a 0 length message */
+ keys[1][0] ^= 0x80;
+
+ /* Encrypt the final result */
+ memcpy(csbcpb->cpb.aes_ecb.key, keys[0], AES_BLOCK_SIZE);
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys[1], sizeof(keys[1]),
+ nx_ctx->ap->sglen);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, out, AES_BLOCK_SIZE,
+ nx_ctx->ap->sglen);
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+out:
+ /* Restore XCBC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
+ memcpy(csbcpb->cpb.aes_xcbc.key, key, AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ return rc;
+}
+
static int nx_xcbc_init(struct shash_desc *desc)
{
struct xcbc_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg;
- u32 to_process, leftover;
+ u32 to_process, leftover, total;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc = 0;
- if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
- /* we've hit the nx chip previously and we're updating again,
- * so copy over the partial digest */
- memcpy(csbcpb->cpb.aes_xcbc.cv,
- csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
- }
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+
+ total = sctx->count + len;
/* 2 cases for total data len:
* 1: <= AES_BLOCK_SIZE: copy into state, return 0
* 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover
*/
- if (len + sctx->count <= AES_BLOCK_SIZE) {
+ if (total <= AES_BLOCK_SIZE) {
memcpy(sctx->buffer + sctx->count, data, len);
sctx->count += len;
goto out;
}
- /* to_process: the AES_BLOCK_SIZE data chunk to process in this
- * update */
- to_process = (sctx->count + len) & ~(AES_BLOCK_SIZE - 1);
- leftover = (sctx->count + len) & (AES_BLOCK_SIZE - 1);
-
- /* the hardware will not accept a 0 byte operation for this algorithm
- * and the operation MUST be finalized to be correct. So if we happen
- * to get an update that falls on a block sized boundary, we must
- * save off the last block to finalize with later. */
- if (!leftover) {
- to_process -= AES_BLOCK_SIZE;
- leftover = AES_BLOCK_SIZE;
- }
-
- if (sctx->count) {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buffer,
- sctx->count, nx_ctx->ap->sglen);
- in_sg = nx_build_sg_list(in_sg, (u8 *)data,
- to_process - sctx->count,
- nx_ctx->ap->sglen);
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+
+ /* to_process: the AES_BLOCK_SIZE data chunk to process in this
+ * update */
+ to_process = min_t(u64, total, nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ /* the hardware will not accept a 0 byte operation for this
+ * algorithm and the operation MUST be finalized to be correct.
+ * So if we happen to get an update that falls on a block sized
+ * boundary, we must save off the last block to finalize with
+ * later. */
+ if (!leftover) {
+ to_process -= AES_BLOCK_SIZE;
+ leftover = AES_BLOCK_SIZE;
+ }
+
+ if (sctx->count) {
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buffer,
+ sctx->count,
+ max_sg_len);
+ }
+ in_sg = nx_build_sg_list(in_sg,
+ (u8 *) data,
+ to_process - sctx->count,
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
- } else {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)data, to_process,
- nx_ctx->ap->sglen);
- nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
- sizeof(struct nx_sg);
- }
- NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ /* we've hit the nx chip previously and we're updating again,
+ * so copy over the partial digest */
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ memcpy(csbcpb->cpb.aes_xcbc.cv,
+ csbcpb->cpb.aes_xcbc.out_cv_mac,
+ AES_BLOCK_SIZE);
+ }
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ atomic_inc(&(nx_ctx->stats->aes_ops));
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
- atomic_inc(&(nx_ctx->stats->aes_ops));
+ total -= to_process;
+ data += to_process - sctx->count;
+ sctx->count = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover > AES_BLOCK_SIZE);
/* copy the leftover back into the state struct */
- memcpy(sctx->buffer, data + len - leftover, leftover);
+ memcpy(sctx->buffer, data, leftover);
sctx->count = leftover;
- /* everything after the first update is continuation */
- NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
+ unsigned long irq_flags;
int rc = 0;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
memcpy(csbcpb->cpb.aes_xcbc.cv,
csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
} else if (sctx->count == 0) {
- /* we've never seen an update, so this is a 0 byte op. The
- * hardware cannot handle a 0 byte op, so just copy out the
- * known 0 byte result. This is cheaper than allocating a
- * software context to do a 0 byte op */
- u8 data[] = { 0x75, 0xf0, 0x25, 0x1d, 0x52, 0x8a, 0xc0, 0x1c,
- 0x45, 0x73, 0xdf, 0xd5, 0x84, 0xd7, 0x9f, 0x29 };
- memcpy(out, data, sizeof(data));
+ /*
+ * we've never seen an update, so this is a 0 byte op. The
+ * hardware cannot handle a 0 byte op, so just ECB to
+ * generate the hash.
+ */
+ rc = nx_xcbc_empty(desc, out);
goto out;
}
memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg;
- u64 to_process, leftover;
+ u64 to_process, leftover, total;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc = 0;
- if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
- /* we've hit the nx chip previously and we're updating again,
- * so copy over the partial digest */
- memcpy(csbcpb->cpb.sha256.input_partial_digest,
- csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
- }
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* 2 cases for total data len:
- * 1: <= SHA256_BLOCK_SIZE: copy into state, return 0
- * 2: > SHA256_BLOCK_SIZE: process X blocks, copy in leftover
+ * 1: < SHA256_BLOCK_SIZE: copy into state, return 0
+ * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover
*/
- if (len + sctx->count < SHA256_BLOCK_SIZE) {
+ total = sctx->count + len;
+ if (total < SHA256_BLOCK_SIZE) {
memcpy(sctx->buf + sctx->count, data, len);
sctx->count += len;
goto out;
}
- /* to_process: the SHA256_BLOCK_SIZE data chunk to process in this
- * update */
- to_process = (sctx->count + len) & ~(SHA256_BLOCK_SIZE - 1);
- leftover = (sctx->count + len) & (SHA256_BLOCK_SIZE - 1);
-
- if (sctx->count) {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
- sctx->count, nx_ctx->ap->sglen);
- in_sg = nx_build_sg_list(in_sg, (u8 *)data,
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the SHA256_BLOCK_SIZE data chunk to process in
+ * this update. This value is also restricted by the sg list
+ * limits.
+ */
+ to_process = min_t(u64, total, nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(SHA256_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ if (sctx->count) {
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buf,
+ sctx->count, max_sg_len);
+ }
+ in_sg = nx_build_sg_list(in_sg, (u8 *) data,
to_process - sctx->count,
- nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
- } else {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)data,
- to_process, nx_ctx->ap->sglen);
- nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
- sizeof(struct nx_sg);
- }
- NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ /*
+ * we've hit the nx chip previously and we're updating
+ * again, so copy over the partial digest.
+ */
+ memcpy(csbcpb->cpb.sha256.input_partial_digest,
+ csbcpb->cpb.sha256.message_digest,
+ SHA256_DIGEST_SIZE);
+ }
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
- atomic_inc(&(nx_ctx->stats->sha256_ops));
+ atomic_inc(&(nx_ctx->stats->sha256_ops));
+ csbcpb->cpb.sha256.message_bit_length += (u64)
+ (csbcpb->cpb.sha256.spbc * 8);
+
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ total -= to_process;
+ data += to_process - sctx->count;
+ sctx->count = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover >= SHA256_BLOCK_SIZE);
/* copy the leftover back into the state struct */
if (leftover)
- memcpy(sctx->buf, data + len - leftover, leftover);
+ memcpy(sctx->buf, data, leftover);
sctx->count = leftover;
-
- csbcpb->cpb.sha256.message_bit_length += (u64)
- (csbcpb->cpb.sha256.spbc * 8);
-
- /* everything after the first update is continuation */
- NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
csbcpb->cpb.sha256.message_bit_length += (u64)(sctx->count * 8);
in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
- sctx->count, nx_ctx->ap->sglen);
+ sctx->count, max_sg_len);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, SHA256_DIGEST_SIZE,
- nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
&(nx_ctx->stats->sha256_bytes));
memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct sha256_state *octx = out;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
octx->count = sctx->count +
(csbcpb->cpb.sha256.message_bit_length / 8);
octx->state[7] = SHA256_H7;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
const struct sha256_state *ictx = in;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg;
- u64 to_process, leftover, spbc_bits;
+ u64 to_process, leftover, total, spbc_bits;
+ u32 max_sg_len;
+ unsigned long irq_flags;
int rc = 0;
- if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
- /* we've hit the nx chip previously and we're updating again,
- * so copy over the partial digest */
- memcpy(csbcpb->cpb.sha512.input_partial_digest,
- csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
- }
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* 2 cases for total data len:
- * 1: <= SHA512_BLOCK_SIZE: copy into state, return 0
- * 2: > SHA512_BLOCK_SIZE: process X blocks, copy in leftover
+ * 1: < SHA512_BLOCK_SIZE: copy into state, return 0
+ * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover
*/
- if ((u64)len + sctx->count[0] < SHA512_BLOCK_SIZE) {
+ total = sctx->count[0] + len;
+ if (total < SHA512_BLOCK_SIZE) {
memcpy(sctx->buf + sctx->count[0], data, len);
sctx->count[0] += len;
goto out;
}
- /* to_process: the SHA512_BLOCK_SIZE data chunk to process in this
- * update */
- to_process = (sctx->count[0] + len) & ~(SHA512_BLOCK_SIZE - 1);
- leftover = (sctx->count[0] + len) & (SHA512_BLOCK_SIZE - 1);
-
- if (sctx->count[0]) {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buf,
- sctx->count[0], nx_ctx->ap->sglen);
- in_sg = nx_build_sg_list(in_sg, (u8 *)data,
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+
+ do {
+ /*
+ * to_process: the SHA512_BLOCK_SIZE data chunk to process in
+ * this update. This value is also restricted by the sg list
+ * limits.
+ */
+ to_process = min_t(u64, total, nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+ to_process = to_process & ~(SHA512_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ if (sctx->count[0]) {
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buf,
+ sctx->count[0], max_sg_len);
+ }
+ in_sg = nx_build_sg_list(in_sg, (u8 *) data,
to_process - sctx->count[0],
- nx_ctx->ap->sglen);
- nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
- sizeof(struct nx_sg);
- } else {
- in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)data,
- to_process, nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
sizeof(struct nx_sg);
- }
- NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ /*
+ * we've hit the nx chip previously and we're updating
+ * again, so copy over the partial digest.
+ */
+ memcpy(csbcpb->cpb.sha512.input_partial_digest,
+ csbcpb->cpb.sha512.message_digest,
+ SHA512_DIGEST_SIZE);
+ }
- if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
- rc = -EINVAL;
- goto out;
- }
-
- rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
- desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
- if (rc)
- goto out;
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->sha512_ops));
+ spbc_bits = csbcpb->cpb.sha512.spbc * 8;
+ csbcpb->cpb.sha512.message_bit_length_lo += spbc_bits;
+ if (csbcpb->cpb.sha512.message_bit_length_lo < spbc_bits)
+ csbcpb->cpb.sha512.message_bit_length_hi++;
+
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
- atomic_inc(&(nx_ctx->stats->sha512_ops));
+ total -= to_process;
+ data += to_process - sctx->count[0];
+ sctx->count[0] = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover >= SHA512_BLOCK_SIZE);
/* copy the leftover back into the state struct */
if (leftover)
- memcpy(sctx->buf, data + len - leftover, leftover);
+ memcpy(sctx->buf, data, leftover);
sctx->count[0] = leftover;
-
- spbc_bits = csbcpb->cpb.sha512.spbc * 8;
- csbcpb->cpb.sha512.message_bit_length_lo += spbc_bits;
- if (csbcpb->cpb.sha512.message_bit_length_lo < spbc_bits)
- csbcpb->cpb.sha512.message_bit_length_hi++;
-
- /* everything after the first update is continuation */
- NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct nx_sg *in_sg, *out_sg;
+ u32 max_sg_len;
u64 count0;
+ unsigned long irq_flags;
int rc;
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
+
if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
/* we've hit the nx chip previously, now we're finalizing,
* so copy over the partial digest */
csbcpb->cpb.sha512.message_bit_length_hi++;
in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, sctx->count[0],
- nx_ctx->ap->sglen);
+ max_sg_len);
out_sg = nx_build_sg_list(nx_ctx->out_sg, out, SHA512_DIGEST_SIZE,
- nx_ctx->ap->sglen);
+ max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return rc;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
struct sha512_state *octx = out;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
/* move message_bit_length (128 bits) into count and convert its value
* to bytes */
octx->state[7] = SHA512_H7;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
const struct sha512_state *ictx = in;
+ unsigned long irq_flags;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
sctx->count[0] = ictx->count[0] & 0x3f;
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
}
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
return 0;
}
do {
rc = vio_h_cop_sync(viodev, op);
- } while ((rc == -EBUSY && !may_sleep && retries--) ||
- (rc == -EBUSY && may_sleep && cond_resched()));
+ } while (rc == -EBUSY && !may_sleep && retries--);
if (rc) {
dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
* have been described (or @sgmax elements have been written), the
* loop ends. min_t is used to ensure @end_addr falls on the same page
* as sg_addr, if not, we need to create another nx_sg element for the
- * data on the next page */
+ * data on the next page.
+ *
+ * Also when using vmalloc'ed data, every time that a system page
+ * boundary is crossed the physical address needs to be re-calculated.
+ */
for (sg = sg_head; sg_len < len; sg++) {
+ u64 next_page;
+
sg->addr = sg_addr;
- sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE), end_addr);
- sg->len = sg_addr - sg->addr;
+ sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE),
+ end_addr);
+
+ next_page = (sg->addr & PAGE_MASK) + PAGE_SIZE;
+ sg->len = min_t(u64, sg_addr, next_page) - sg->addr;
sg_len += sg->len;
+ if (sg_addr >= next_page &&
+ is_vmalloc_addr(start_addr + sg_len)) {
+ sg_addr = page_to_phys(vmalloc_to_page(
+ start_addr + sg_len));
+ end_addr = sg_addr + len - sg_len;
+ }
+
if ((sg - sg_head) == sgmax) {
pr_err("nx: scatter/gather list overflow, pid: %d\n",
current->pid);
* @dst: destination scatterlist
* @src: source scatterlist
* @nbytes: length of data described in the scatterlists
+ * @offset: number of bytes to fast-forward past at the beginning of
+ * scatterlists.
* @iv: destination for the iv data, if the algorithm requires it
*
* This is common code shared by all the AES algorithms. It uses the block
struct scatterlist *dst,
struct scatterlist *src,
unsigned int nbytes,
+ unsigned int offset,
u8 *iv)
{
struct nx_sg *nx_insg = nx_ctx->in_sg;
if (iv)
memcpy(iv, desc->info, AES_BLOCK_SIZE);
- nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen, src, 0, nbytes);
- nx_outsg = nx_walk_and_build(nx_outsg, nx_ctx->ap->sglen, dst, 0, nbytes);
+ nx_insg = nx_walk_and_build(nx_insg, nx_ctx->ap->sglen, src,
+ offset, nbytes);
+ nx_outsg = nx_walk_and_build(nx_outsg, nx_ctx->ap->sglen, dst,
+ offset, nbytes);
/* these lengths should be negative, which will indicate to phyp that
* the input and output parameters are scatterlists, not linear
*/
void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
{
+ spin_lock_init(&nx_ctx->lock);
memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
};
struct nx_crypto_ctx {
+ spinlock_t lock; /* synchronize access to the context */
void *kmem; /* unaligned, kmalloc'd buffer */
size_t kmem_len; /* length of kmem */
struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */
struct nx_sg *nx_build_sg_list(struct nx_sg *, u8 *, unsigned int, u32);
int nx_build_sg_lists(struct nx_crypto_ctx *, struct blkcipher_desc *,
struct scatterlist *, struct scatterlist *, unsigned int,
- u8 *);
+ unsigned int, u8 *);
struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
struct scatterlist *, unsigned int,
unsigned int);
*
*/
-#define pr_fmt(fmt) "%s: " fmt, __func__
+#define pr_fmt(fmt) "%20s: " fmt, __func__
+#define prn(num) pr_debug(#num "=%d\n", num)
+#define prx(num) pr_debug(#num "=%x\n", num)
#include <linux/err.h>
#include <linux/module.h>
#define DST_MAXBURST 4
#define DMA_MIN (DST_MAXBURST * sizeof(u32))
+#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
+
/* OMAP TRM gives bitfields as start:end, where start is the higher bit
number. For example 7:0 */
#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end))
#define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04))
+#define AES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs)
+#define AES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs)
+#define AES_REG_IRQ_DATA_IN BIT(1)
+#define AES_REG_IRQ_DATA_OUT BIT(2)
#define DEFAULT_TIMEOUT (5*HZ)
#define FLAGS_MODE_MASK 0x000f
#define FLAGS_FAST BIT(5)
#define FLAGS_BUSY BIT(6)
+#define AES_BLOCK_WORDS (AES_BLOCK_SIZE >> 2)
+
struct omap_aes_ctx {
struct omap_aes_dev *dd;
u32 data_ofs;
u32 rev_ofs;
u32 mask_ofs;
+ u32 irq_enable_ofs;
+ u32 irq_status_ofs;
u32 dma_enable_in;
u32 dma_enable_out;
struct tasklet_struct queue_task;
struct ablkcipher_request *req;
+
+ /*
+ * total is used by PIO mode for book keeping so introduce
+ * variable total_save as need it to calc page_order
+ */
size_t total;
+ size_t total_save;
+
struct scatterlist *in_sg;
- struct scatterlist in_sgl;
- size_t in_offset;
struct scatterlist *out_sg;
+
+ /* Buffers for copying for unaligned cases */
+ struct scatterlist in_sgl;
struct scatterlist out_sgl;
- size_t out_offset;
+ struct scatterlist *orig_out;
+ int sgs_copied;
- size_t buflen;
- void *buf_in;
- size_t dma_size;
+ struct scatter_walk in_walk;
+ struct scatter_walk out_walk;
int dma_in;
struct dma_chan *dma_lch_in;
- dma_addr_t dma_addr_in;
- void *buf_out;
int dma_out;
struct dma_chan *dma_lch_out;
- dma_addr_t dma_addr_out;
-
+ int in_sg_len;
+ int out_sg_len;
+ int pio_only;
const struct omap_aes_pdata *pdata;
};
static LIST_HEAD(dev_list);
static DEFINE_SPINLOCK(list_lock);
+#ifdef DEBUG
+#define omap_aes_read(dd, offset) \
+({ \
+ int _read_ret; \
+ _read_ret = __raw_readl(dd->io_base + offset); \
+ pr_debug("omap_aes_read(" #offset "=%#x)= %#x\n", \
+ offset, _read_ret); \
+ _read_ret; \
+})
+#else
static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
{
return __raw_readl(dd->io_base + offset);
}
+#endif
+#ifdef DEBUG
+#define omap_aes_write(dd, offset, value) \
+ do { \
+ pr_debug("omap_aes_write(" #offset "=%#x) value=%#x\n", \
+ offset, value); \
+ __raw_writel(value, dd->io_base + offset); \
+ } while (0)
+#else
static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
u32 value)
{
__raw_writel(value, dd->io_base + offset);
}
+#endif
static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
u32 value, u32 mask)
dd->dma_lch_out = NULL;
dd->dma_lch_in = NULL;
- dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
- dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
- dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
- dd->buflen &= ~(AES_BLOCK_SIZE - 1);
-
- if (!dd->buf_in || !dd->buf_out) {
- dev_err(dd->dev, "unable to alloc pages.\n");
- goto err_alloc;
- }
-
- /* MAP here */
- dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
- DMA_TO_DEVICE);
- if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
- dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
- err = -EINVAL;
- goto err_map_in;
- }
-
- dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
- DMA_FROM_DEVICE);
- if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
- dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
- err = -EINVAL;
- goto err_map_out;
- }
-
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
err_dma_out:
dma_release_channel(dd->dma_lch_in);
err_dma_in:
- dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
- DMA_FROM_DEVICE);
-err_map_out:
- dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
-err_map_in:
- free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
- free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
-err_alloc:
if (err)
pr_err("error: %d\n", err);
return err;
{
dma_release_channel(dd->dma_lch_out);
dma_release_channel(dd->dma_lch_in);
- dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
- DMA_FROM_DEVICE);
- dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
- free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
- free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
}
static void sg_copy_buf(void *buf, struct scatterlist *sg,
scatterwalk_done(&walk, out, 0);
}
-static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
- size_t buflen, size_t total, int out)
-{
- unsigned int count, off = 0;
-
- while (buflen && total) {
- count = min((*sg)->length - *offset, total);
- count = min(count, buflen);
-
- if (!count)
- return off;
-
- /*
- * buflen and total are AES_BLOCK_SIZE size aligned,
- * so count should be also aligned
- */
-
- sg_copy_buf(buf + off, *sg, *offset, count, out);
-
- off += count;
- buflen -= count;
- *offset += count;
- total -= count;
-
- if (*offset == (*sg)->length) {
- *sg = sg_next(*sg);
- if (*sg)
- *offset = 0;
- else
- total = 0;
- }
- }
-
- return off;
-}
-
static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
- struct scatterlist *in_sg, struct scatterlist *out_sg)
+ struct scatterlist *in_sg, struct scatterlist *out_sg,
+ int in_sg_len, int out_sg_len)
{
struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
struct omap_aes_dev *dd = ctx->dd;
struct dma_async_tx_descriptor *tx_in, *tx_out;
struct dma_slave_config cfg;
- dma_addr_t dma_addr_in = sg_dma_address(in_sg);
- int ret, length = sg_dma_len(in_sg);
+ int ret;
- pr_debug("len: %d\n", length);
+ if (dd->pio_only) {
+ scatterwalk_start(&dd->in_walk, dd->in_sg);
+ scatterwalk_start(&dd->out_walk, dd->out_sg);
- dd->dma_size = length;
+ /* Enable DATAIN interrupt and let it take
+ care of the rest */
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
+ return 0;
+ }
- if (!(dd->flags & FLAGS_FAST))
- dma_sync_single_for_device(dd->dev, dma_addr_in, length,
- DMA_TO_DEVICE);
+ dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
memset(&cfg, 0, sizeof(cfg));
return ret;
}
- tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, 1,
+ tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
DMA_MEM_TO_DEV,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!tx_in) {
return ret;
}
- tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, 1,
+ tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,
DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!tx_out) {
dma_async_issue_pending(dd->dma_lch_out);
/* start DMA */
- dd->pdata->trigger(dd, length);
+ dd->pdata->trigger(dd, dd->total);
return 0;
}
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
crypto_ablkcipher_reqtfm(dd->req));
- int err, fast = 0, in, out;
- size_t count;
- dma_addr_t addr_in, addr_out;
- struct scatterlist *in_sg, *out_sg;
- int len32;
+ int err;
pr_debug("total: %d\n", dd->total);
- if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
- /* check for alignment */
- in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
- out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
-
- fast = in && out;
- }
-
- if (fast) {
- count = min(dd->total, sg_dma_len(dd->in_sg));
- count = min(count, sg_dma_len(dd->out_sg));
-
- if (count != dd->total) {
- pr_err("request length != buffer length\n");
- return -EINVAL;
- }
-
- pr_debug("fast\n");
-
- err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+ if (!dd->pio_only) {
+ err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
+ DMA_TO_DEVICE);
if (!err) {
dev_err(dd->dev, "dma_map_sg() error\n");
return -EINVAL;
}
- err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
+ err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
if (!err) {
dev_err(dd->dev, "dma_map_sg() error\n");
- dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
return -EINVAL;
}
-
- addr_in = sg_dma_address(dd->in_sg);
- addr_out = sg_dma_address(dd->out_sg);
-
- in_sg = dd->in_sg;
- out_sg = dd->out_sg;
-
- dd->flags |= FLAGS_FAST;
-
- } else {
- /* use cache buffers */
- count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
- dd->buflen, dd->total, 0);
-
- len32 = DIV_ROUND_UP(count, DMA_MIN) * DMA_MIN;
-
- /*
- * The data going into the AES module has been copied
- * to a local buffer and the data coming out will go
- * into a local buffer so set up local SG entries for
- * both.
- */
- sg_init_table(&dd->in_sgl, 1);
- dd->in_sgl.offset = dd->in_offset;
- sg_dma_len(&dd->in_sgl) = len32;
- sg_dma_address(&dd->in_sgl) = dd->dma_addr_in;
-
- sg_init_table(&dd->out_sgl, 1);
- dd->out_sgl.offset = dd->out_offset;
- sg_dma_len(&dd->out_sgl) = len32;
- sg_dma_address(&dd->out_sgl) = dd->dma_addr_out;
-
- in_sg = &dd->in_sgl;
- out_sg = &dd->out_sgl;
-
- addr_in = dd->dma_addr_in;
- addr_out = dd->dma_addr_out;
-
- dd->flags &= ~FLAGS_FAST;
-
}
- dd->total -= count;
-
- err = omap_aes_crypt_dma(tfm, in_sg, out_sg);
- if (err) {
- dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
- dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
+ err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,
+ dd->out_sg_len);
+ if (err && !dd->pio_only) {
+ dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
+ dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
}
return err;
static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
{
int err = 0;
- size_t count;
pr_debug("total: %d\n", dd->total);
dmaengine_terminate_all(dd->dma_lch_in);
dmaengine_terminate_all(dd->dma_lch_out);
- if (dd->flags & FLAGS_FAST) {
- dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
- dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
- } else {
- dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
- dd->dma_size, DMA_FROM_DEVICE);
-
- /* copy data */
- count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
- dd->buflen, dd->dma_size, 1);
- if (count != dd->dma_size) {
- err = -EINVAL;
- pr_err("not all data converted: %u\n", count);
- }
+ return err;
+}
+
+int omap_aes_check_aligned(struct scatterlist *sg)
+{
+ while (sg) {
+ if (!IS_ALIGNED(sg->offset, 4))
+ return -1;
+ if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
+ return -1;
+ sg = sg_next(sg);
}
+ return 0;
+}
- return err;
+int omap_aes_copy_sgs(struct omap_aes_dev *dd)
+{
+ void *buf_in, *buf_out;
+ int pages;
+
+ pages = get_order(dd->total);
+
+ buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
+ buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
+
+ if (!buf_in || !buf_out) {
+ pr_err("Couldn't allocated pages for unaligned cases.\n");
+ return -1;
+ }
+
+ dd->orig_out = dd->out_sg;
+
+ sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
+
+ sg_init_table(&dd->in_sgl, 1);
+ sg_set_buf(&dd->in_sgl, buf_in, dd->total);
+ dd->in_sg = &dd->in_sgl;
+
+ sg_init_table(&dd->out_sgl, 1);
+ sg_set_buf(&dd->out_sgl, buf_out, dd->total);
+ dd->out_sg = &dd->out_sgl;
+
+ return 0;
}
static int omap_aes_handle_queue(struct omap_aes_dev *dd,
/* assign new request to device */
dd->req = req;
dd->total = req->nbytes;
- dd->in_offset = 0;
+ dd->total_save = req->nbytes;
dd->in_sg = req->src;
- dd->out_offset = 0;
dd->out_sg = req->dst;
+ if (omap_aes_check_aligned(dd->in_sg) ||
+ omap_aes_check_aligned(dd->out_sg)) {
+ if (omap_aes_copy_sgs(dd))
+ pr_err("Failed to copy SGs for unaligned cases\n");
+ dd->sgs_copied = 1;
+ } else {
+ dd->sgs_copied = 0;
+ }
+
+ dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total);
+ dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total);
+ BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0);
+
rctx = ablkcipher_request_ctx(req);
ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
rctx->mode &= FLAGS_MODE_MASK;
static void omap_aes_done_task(unsigned long data)
{
struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
- int err;
-
- pr_debug("enter\n");
+ void *buf_in, *buf_out;
+ int pages;
+
+ pr_debug("enter done_task\n");
+
+ if (!dd->pio_only) {
+ dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
+ dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
+ dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
+ DMA_FROM_DEVICE);
+ omap_aes_crypt_dma_stop(dd);
+ }
- err = omap_aes_crypt_dma_stop(dd);
+ if (dd->sgs_copied) {
+ buf_in = sg_virt(&dd->in_sgl);
+ buf_out = sg_virt(&dd->out_sgl);
- err = dd->err ? : err;
+ sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
- if (dd->total && !err) {
- err = omap_aes_crypt_dma_start(dd);
- if (!err)
- return; /* DMA started. Not fininishing. */
+ pages = get_order(dd->total_save);
+ free_pages((unsigned long)buf_in, pages);
+ free_pages((unsigned long)buf_out, pages);
}
- omap_aes_finish_req(dd, err);
+ omap_aes_finish_req(dd, 0);
omap_aes_handle_queue(dd, NULL);
pr_debug("exit\n");
.data_ofs = 0x60,
.rev_ofs = 0x80,
.mask_ofs = 0x84,
+ .irq_status_ofs = 0x8c,
+ .irq_enable_ofs = 0x90,
.dma_enable_in = BIT(5),
.dma_enable_out = BIT(6),
.major_mask = 0x0700,
.minor_shift = 0,
};
+static irqreturn_t omap_aes_irq(int irq, void *dev_id)
+{
+ struct omap_aes_dev *dd = dev_id;
+ u32 status, i;
+ u32 *src, *dst;
+
+ status = omap_aes_read(dd, AES_REG_IRQ_STATUS(dd));
+ if (status & AES_REG_IRQ_DATA_IN) {
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
+
+ BUG_ON(!dd->in_sg);
+
+ BUG_ON(_calc_walked(in) > dd->in_sg->length);
+
+ src = sg_virt(dd->in_sg) + _calc_walked(in);
+
+ for (i = 0; i < AES_BLOCK_WORDS; i++) {
+ omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src);
+
+ scatterwalk_advance(&dd->in_walk, 4);
+ if (dd->in_sg->length == _calc_walked(in)) {
+ dd->in_sg = scatterwalk_sg_next(dd->in_sg);
+ if (dd->in_sg) {
+ scatterwalk_start(&dd->in_walk,
+ dd->in_sg);
+ src = sg_virt(dd->in_sg) +
+ _calc_walked(in);
+ }
+ } else {
+ src++;
+ }
+ }
+
+ /* Clear IRQ status */
+ status &= ~AES_REG_IRQ_DATA_IN;
+ omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
+
+ /* Enable DATA_OUT interrupt */
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x4);
+
+ } else if (status & AES_REG_IRQ_DATA_OUT) {
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x0);
+
+ BUG_ON(!dd->out_sg);
+
+ BUG_ON(_calc_walked(out) > dd->out_sg->length);
+
+ dst = sg_virt(dd->out_sg) + _calc_walked(out);
+
+ for (i = 0; i < AES_BLOCK_WORDS; i++) {
+ *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
+ scatterwalk_advance(&dd->out_walk, 4);
+ if (dd->out_sg->length == _calc_walked(out)) {
+ dd->out_sg = scatterwalk_sg_next(dd->out_sg);
+ if (dd->out_sg) {
+ scatterwalk_start(&dd->out_walk,
+ dd->out_sg);
+ dst = sg_virt(dd->out_sg) +
+ _calc_walked(out);
+ }
+ } else {
+ dst++;
+ }
+ }
+
+ dd->total -= AES_BLOCK_SIZE;
+
+ BUG_ON(dd->total < 0);
+
+ /* Clear IRQ status */
+ status &= ~AES_REG_IRQ_DATA_OUT;
+ omap_aes_write(dd, AES_REG_IRQ_STATUS(dd), status);
+
+ if (!dd->total)
+ /* All bytes read! */
+ tasklet_schedule(&dd->done_task);
+ else
+ /* Enable DATA_IN interrupt for next block */
+ omap_aes_write(dd, AES_REG_IRQ_ENABLE(dd), 0x2);
+ }
+
+ return IRQ_HANDLED;
+}
+
static const struct of_device_id omap_aes_of_match[] = {
{
.compatible = "ti,omap2-aes",
struct omap_aes_dev *dd;
struct crypto_alg *algp;
struct resource res;
- int err = -ENOMEM, i, j;
+ int err = -ENOMEM, i, j, irq = -1;
u32 reg;
- dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
+ dd = devm_kzalloc(dev, sizeof(struct omap_aes_dev), GFP_KERNEL);
if (dd == NULL) {
dev_err(dev, "unable to alloc data struct.\n");
goto err_data;
tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
err = omap_aes_dma_init(dd);
- if (err)
- goto err_dma;
+ if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) {
+ dd->pio_only = 1;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ dev_err(dev, "can't get IRQ resource\n");
+ goto err_irq;
+ }
+
+ err = devm_request_irq(dev, irq, omap_aes_irq, 0,
+ dev_name(dev), dd);
+ if (err) {
+ dev_err(dev, "Unable to grab omap-aes IRQ\n");
+ goto err_irq;
+ }
+ }
+
INIT_LIST_HEAD(&dd->list);
spin_lock(&list_lock);
for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
crypto_unregister_alg(
&dd->pdata->algs_info[i].algs_list[j]);
- omap_aes_dma_cleanup(dd);
-err_dma:
+ if (!dd->pio_only)
+ omap_aes_dma_cleanup(dd);
+err_irq:
tasklet_kill(&dd->done_task);
tasklet_kill(&dd->queue_task);
pm_runtime_disable(dev);
err_res:
- kfree(dd);
dd = NULL;
err_data:
dev_err(dev, "initialization failed.\n");
tasklet_kill(&dd->queue_task);
omap_aes_dma_cleanup(dd);
pm_runtime_disable(dd->dev);
- kfree(dd);
dd = NULL;
return 0;
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
-#define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
#define MD5_DIGEST_SIZE 16
-#define DST_MAXBURST 16
-#define DMA_MIN (DST_MAXBURST * sizeof(u32))
-
#define SHA_REG_IDIGEST(dd, x) ((dd)->pdata->idigest_ofs + ((x)*0x04))
#define SHA_REG_DIN(dd, x) ((dd)->pdata->din_ofs + ((x) * 0x04))
#define SHA_REG_DIGCNT(dd) ((dd)->pdata->digcnt_ofs)
-#define SHA_REG_ODIGEST(x) (0x00 + ((x) * 0x04))
+#define SHA_REG_ODIGEST(dd, x) ((dd)->pdata->odigest_ofs + (x * 0x04))
#define SHA_REG_CTRL 0x18
#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
#define SHA_REG_SYSSTATUS(dd) ((dd)->pdata->sysstatus_ofs)
#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
-#define SHA_REG_MODE 0x44
+#define SHA_REG_MODE(dd) ((dd)->pdata->mode_ofs)
#define SHA_REG_MODE_HMAC_OUTER_HASH (1 << 7)
#define SHA_REG_MODE_HMAC_KEY_PROC (1 << 5)
#define SHA_REG_MODE_CLOSE_HASH (1 << 4)
#define SHA_REG_MODE_ALGO_CONSTANT (1 << 3)
-#define SHA_REG_MODE_ALGO_MASK (3 << 1)
-#define SHA_REG_MODE_ALGO_MD5_128 (0 << 1)
-#define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1)
-#define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1)
-#define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1)
-#define SHA_REG_LENGTH 0x48
+#define SHA_REG_MODE_ALGO_MASK (7 << 0)
+#define SHA_REG_MODE_ALGO_MD5_128 (0 << 1)
+#define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1)
+#define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1)
+#define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1)
+#define SHA_REG_MODE_ALGO_SHA2_384 (1 << 0)
+#define SHA_REG_MODE_ALGO_SHA2_512 (3 << 0)
+
+#define SHA_REG_LENGTH(dd) ((dd)->pdata->length_ofs)
#define SHA_REG_IRQSTATUS 0x118
#define SHA_REG_IRQSTATUS_CTX_RDY (1 << 3)
#define FLAGS_SG 17
#define FLAGS_MODE_SHIFT 18
-#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 \
- << (FLAGS_MODE_SHIFT - 1))
-#define FLAGS_HMAC 20
-#define FLAGS_ERROR 21
+#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA384 (SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT)
+#define FLAGS_MODE_SHA512 (SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT)
+
+#define FLAGS_HMAC 21
+#define FLAGS_ERROR 22
#define OP_UPDATE 1
#define OP_FINAL 2
unsigned long flags;
unsigned long op;
- u8 digest[SHA256_DIGEST_SIZE] OMAP_ALIGNED;
+ u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
size_t digcnt;
size_t bufcnt;
size_t buflen;
struct omap_sham_hmac_ctx {
struct crypto_shash *shash;
- u8 ipad[SHA1_MD5_BLOCK_SIZE] OMAP_ALIGNED;
- u8 opad[SHA1_MD5_BLOCK_SIZE] OMAP_ALIGNED;
+ u8 ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
+ u8 opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
};
struct omap_sham_ctx {
u32 rev_ofs;
u32 mask_ofs;
u32 sysstatus_ofs;
+ u32 mode_ofs;
+ u32 length_ofs;
u32 major_mask;
u32 major_shift;
unsigned int dma;
struct dma_chan *dma_lch;
struct tasklet_struct done_task;
+ u8 polling_mode;
unsigned long flags;
struct crypto_queue queue;
for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
if (out)
opad[i] = omap_sham_read(dd,
- SHA_REG_ODIGEST(i));
+ SHA_REG_ODIGEST(dd, i));
else
- omap_sham_write(dd, SHA_REG_ODIGEST(i),
+ omap_sham_write(dd, SHA_REG_ODIGEST(dd, i),
opad[i]);
}
}
case FLAGS_MODE_SHA256:
d = SHA256_DIGEST_SIZE / sizeof(u32);
break;
+ case FLAGS_MODE_SHA384:
+ d = SHA384_DIGEST_SIZE / sizeof(u32);
+ break;
+ case FLAGS_MODE_SHA512:
+ d = SHA512_DIGEST_SIZE / sizeof(u32);
+ break;
default:
d = 0;
}
return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
}
+static int get_block_size(struct omap_sham_reqctx *ctx)
+{
+ int d;
+
+ switch (ctx->flags & FLAGS_MODE_MASK) {
+ case FLAGS_MODE_MD5:
+ case FLAGS_MODE_SHA1:
+ d = SHA1_BLOCK_SIZE;
+ break;
+ case FLAGS_MODE_SHA224:
+ case FLAGS_MODE_SHA256:
+ d = SHA256_BLOCK_SIZE;
+ break;
+ case FLAGS_MODE_SHA384:
+ case FLAGS_MODE_SHA512:
+ d = SHA512_BLOCK_SIZE;
+ break;
+ default:
+ d = 0;
+ }
+
+ return d;
+}
+
static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
u32 *value, int count)
{
* CLOSE_HASH only for the last one. Note that flags mode bits
* correspond to algorithm encoding in mode register.
*/
- val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT - 1);
+ val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);
if (!ctx->digcnt) {
struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
struct omap_sham_hmac_ctx *bctx = tctx->base;
+ int bs, nr_dr;
val |= SHA_REG_MODE_ALGO_CONSTANT;
if (ctx->flags & BIT(FLAGS_HMAC)) {
+ bs = get_block_size(ctx);
+ nr_dr = bs / (2 * sizeof(u32));
val |= SHA_REG_MODE_HMAC_KEY_PROC;
- omap_sham_write_n(dd, SHA_REG_ODIGEST(0),
- (u32 *)bctx->ipad,
- SHA1_BLOCK_SIZE / sizeof(u32));
- ctx->digcnt += SHA1_BLOCK_SIZE;
+ omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0),
+ (u32 *)bctx->ipad, nr_dr);
+ omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0),
+ (u32 *)bctx->ipad + nr_dr, nr_dr);
+ ctx->digcnt += bs;
}
}
SHA_REG_MODE_HMAC_KEY_PROC;
dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
- omap_sham_write_mask(dd, SHA_REG_MODE, val, mask);
+ omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask);
omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
omap_sham_write_mask(dd, SHA_REG_MASK(dd),
SHA_REG_MASK_IT_EN |
static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
{
- omap_sham_write(dd, SHA_REG_LENGTH, length);
+ omap_sham_write(dd, SHA_REG_LENGTH(dd), length);
}
static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
size_t length, int final)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- int count, len32;
+ int count, len32, bs32, offset = 0;
const u32 *buffer = (const u32 *)buf;
dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
/* should be non-zero before next lines to disable clocks later */
ctx->digcnt += length;
- if (dd->pdata->poll_irq(dd))
- return -ETIMEDOUT;
-
if (final)
set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
set_bit(FLAGS_CPU, &dd->flags);
len32 = DIV_ROUND_UP(length, sizeof(u32));
+ bs32 = get_block_size(ctx) / sizeof(u32);
- for (count = 0; count < len32; count++)
- omap_sham_write(dd, SHA_REG_DIN(dd, count), buffer[count]);
+ while (len32) {
+ if (dd->pdata->poll_irq(dd))
+ return -ETIMEDOUT;
+
+ for (count = 0; count < min(len32, bs32); count++, offset++)
+ omap_sham_write(dd, SHA_REG_DIN(dd, count),
+ buffer[offset]);
+ len32 -= min(len32, bs32);
+ }
return -EINPROGRESS;
}
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
struct dma_async_tx_descriptor *tx;
struct dma_slave_config cfg;
- int len32, ret;
+ int len32, ret, dma_min = get_block_size(ctx);
dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
ctx->digcnt, length, final);
cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
- cfg.dst_maxburst = DST_MAXBURST;
+ cfg.dst_maxburst = dma_min / DMA_SLAVE_BUSWIDTH_4_BYTES;
ret = dmaengine_slave_config(dd->dma_lch, &cfg);
if (ret) {
return ret;
}
- len32 = DIV_ROUND_UP(length, DMA_MIN) * DMA_MIN;
+ len32 = DIV_ROUND_UP(length, dma_min) * dma_min;
if (is_sg) {
/*
/* Start address alignment */
#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
/* SHA1 block size alignment */
-#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
+#define SG_SA(sg, bs) (IS_ALIGNED(sg->length, bs))
static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
unsigned int length, final, tail;
struct scatterlist *sg;
- int ret;
+ int ret, bs;
if (!ctx->total)
return 0;
* the dmaengine infrastructure will calculate that it needs
* to transfer 0 frames which ultimately fails.
*/
- if (ctx->total < (DST_MAXBURST * sizeof(u32)))
+ if (ctx->total < get_block_size(ctx))
return omap_sham_update_dma_slow(dd);
dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
ctx->digcnt, ctx->bufcnt, ctx->total);
sg = ctx->sg;
+ bs = get_block_size(ctx);
if (!SG_AA(sg))
return omap_sham_update_dma_slow(dd);
- if (!sg_is_last(sg) && !SG_SA(sg))
- /* size is not SHA1_BLOCK_SIZE aligned */
+ if (!sg_is_last(sg) && !SG_SA(sg, bs))
+ /* size is not BLOCK_SIZE aligned */
return omap_sham_update_dma_slow(dd);
length = min(ctx->total, sg->length);
if (sg_is_last(sg)) {
if (!(ctx->flags & BIT(FLAGS_FINUP))) {
- /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
- tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
+ /* not last sg must be BLOCK_SIZE aligned */
+ tail = length & (bs - 1);
/* without finup() we need one block to close hash */
if (!tail)
- tail = SHA1_MD5_BLOCK_SIZE;
+ tail = bs;
length -= tail;
}
}
static int omap_sham_update_cpu(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- int bufcnt;
+ int bufcnt, final;
+
+ if (!ctx->total)
+ return 0;
omap_sham_append_sg(ctx);
+
+ final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+
+ dev_dbg(dd->dev, "cpu: bufcnt: %u, digcnt: %d, final: %d\n",
+ ctx->bufcnt, ctx->digcnt, final);
+
bufcnt = ctx->bufcnt;
ctx->bufcnt = 0;
- return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
+ return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final);
}
static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
struct omap_sham_dev *dd = NULL, *tmp;
+ int bs = 0;
spin_lock_bh(&sham.lock);
if (!tctx->dd) {
switch (crypto_ahash_digestsize(tfm)) {
case MD5_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_MD5;
+ bs = SHA1_BLOCK_SIZE;
break;
case SHA1_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_SHA1;
+ bs = SHA1_BLOCK_SIZE;
break;
case SHA224_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_SHA224;
+ bs = SHA224_BLOCK_SIZE;
break;
case SHA256_DIGEST_SIZE:
ctx->flags |= FLAGS_MODE_SHA256;
+ bs = SHA256_BLOCK_SIZE;
+ break;
+ case SHA384_DIGEST_SIZE:
+ ctx->flags |= FLAGS_MODE_SHA384;
+ bs = SHA384_BLOCK_SIZE;
+ break;
+ case SHA512_DIGEST_SIZE:
+ ctx->flags |= FLAGS_MODE_SHA512;
+ bs = SHA512_BLOCK_SIZE;
break;
}
if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
struct omap_sham_hmac_ctx *bctx = tctx->base;
- memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
- ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
+ memcpy(ctx->buffer, bctx->ipad, bs);
+ ctx->bufcnt = bs;
}
ctx->flags |= BIT(FLAGS_HMAC);
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
int err = 0, use_dma = 1;
- if (ctx->bufcnt <= DMA_MIN)
- /* faster to handle last block with cpu */
+ if ((ctx->bufcnt <= get_block_size(ctx)) || dd->polling_mode)
+ /*
+ * faster to handle last block with cpu or
+ * use cpu when dma is not present.
+ */
use_dma = 0;
if (use_dma)
static int omap_sham_update(struct ahash_request *req)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = ctx->dd;
+ int bs = get_block_size(ctx);
if (!req->nbytes)
return 0;
*/
omap_sham_append_sg(ctx);
return 0;
- } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) {
+ } else if ((ctx->bufcnt + ctx->total <= bs) ||
+ dd->polling_mode) {
/*
- * faster to use CPU for short transfers
- */
+ * faster to use CPU for short transfers or
+ * use cpu when dma is not present.
+ */
ctx->flags |= BIT(FLAGS_CPU);
}
} else if (ctx->bufcnt + ctx->total < ctx->buflen) {
return omap_sham_cra_init_alg(tfm, "md5");
}
+static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, "sha384");
+}
+
+static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, "sha512");
+}
+
static void omap_sham_cra_exit(struct crypto_tfm *tfm)
{
struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
},
};
+static struct ahash_alg algs_sha384_sha512[] = {
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "omap-sha384",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "omap-sha512",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .setkey = omap_sham_setkey,
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "omap-hmac-sha384",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx) +
+ sizeof(struct omap_sham_hmac_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_sha384_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .setkey = omap_sham_setkey,
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "omap-hmac-sha512",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx) +
+ sizeof(struct omap_sham_hmac_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_sha512_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+};
+
static void omap_sham_done_task(unsigned long data)
{
struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
}
if (test_bit(FLAGS_CPU, &dd->flags)) {
- if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
- goto finish;
+ if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
+ /* hash or semi-hash ready */
+ err = omap_sham_update_cpu(dd);
+ if (err != -EINPROGRESS)
+ goto finish;
+ }
} else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
omap_sham_update_dma_stop(dd);
.poll_irq = omap_sham_poll_irq_omap4,
.intr_hdlr = omap_sham_irq_omap4,
.idigest_ofs = 0x020,
+ .odigest_ofs = 0x0,
.din_ofs = 0x080,
.digcnt_ofs = 0x040,
.rev_ofs = 0x100,
.mask_ofs = 0x110,
.sysstatus_ofs = 0x114,
+ .mode_ofs = 0x44,
+ .length_ofs = 0x48,
+ .major_mask = 0x0700,
+ .major_shift = 8,
+ .minor_mask = 0x003f,
+ .minor_shift = 0,
+};
+
+static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = {
+ {
+ .algs_list = algs_sha1_md5,
+ .size = ARRAY_SIZE(algs_sha1_md5),
+ },
+ {
+ .algs_list = algs_sha224_sha256,
+ .size = ARRAY_SIZE(algs_sha224_sha256),
+ },
+ {
+ .algs_list = algs_sha384_sha512,
+ .size = ARRAY_SIZE(algs_sha384_sha512),
+ },
+};
+
+static const struct omap_sham_pdata omap_sham_pdata_omap5 = {
+ .algs_info = omap_sham_algs_info_omap5,
+ .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap5),
+ .flags = BIT(FLAGS_AUTO_XOR),
+ .digest_size = SHA512_DIGEST_SIZE,
+ .copy_hash = omap_sham_copy_hash_omap4,
+ .write_ctrl = omap_sham_write_ctrl_omap4,
+ .trigger = omap_sham_trigger_omap4,
+ .poll_irq = omap_sham_poll_irq_omap4,
+ .intr_hdlr = omap_sham_irq_omap4,
+ .idigest_ofs = 0x240,
+ .odigest_ofs = 0x200,
+ .din_ofs = 0x080,
+ .digcnt_ofs = 0x280,
+ .rev_ofs = 0x100,
+ .mask_ofs = 0x110,
+ .sysstatus_ofs = 0x114,
+ .mode_ofs = 0x284,
+ .length_ofs = 0x288,
.major_mask = 0x0700,
.major_shift = 8,
.minor_mask = 0x003f,
.compatible = "ti,omap4-sham",
.data = &omap_sham_pdata_omap4,
},
+ {
+ .compatible = "ti,omap5-sham",
+ .data = &omap_sham_pdata_omap5,
+ },
{},
};
MODULE_DEVICE_TABLE(of, omap_sham_of_match);
int err, i, j;
u32 rev;
- dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL);
+ dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL);
if (dd == NULL) {
dev_err(dev, "unable to alloc data struct.\n");
err = -ENOMEM;
err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
omap_sham_get_res_pdev(dd, pdev, &res);
if (err)
- goto res_err;
+ goto data_err;
dd->io_base = devm_ioremap_resource(dev, &res);
if (IS_ERR(dd->io_base)) {
err = PTR_ERR(dd->io_base);
- goto res_err;
+ goto data_err;
}
dd->phys_base = res.start;
- err = request_irq(dd->irq, dd->pdata->intr_hdlr, IRQF_TRIGGER_LOW,
- dev_name(dev), dd);
+ err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr,
+ IRQF_TRIGGER_NONE, dev_name(dev), dd);
if (err) {
- dev_err(dev, "unable to request irq.\n");
- goto res_err;
+ dev_err(dev, "unable to request irq %d, err = %d\n",
+ dd->irq, err);
+ goto data_err;
}
dma_cap_zero(mask);
dd->dma_lch = dma_request_slave_channel_compat(mask, omap_dma_filter_fn,
&dd->dma, dev, "rx");
if (!dd->dma_lch) {
- dev_err(dev, "unable to obtain RX DMA engine channel %u\n",
- dd->dma);
- err = -ENXIO;
- goto dma_err;
+ dd->polling_mode = 1;
+ dev_dbg(dev, "using polling mode instead of dma\n");
}
dd->flags |= dd->pdata->flags;
&dd->pdata->algs_info[i].algs_list[j]);
pm_runtime_disable(dev);
dma_release_channel(dd->dma_lch);
-dma_err:
- free_irq(dd->irq, dd);
-res_err:
- kfree(dd);
- dd = NULL;
data_err:
dev_err(dev, "initialization failed.\n");
tasklet_kill(&dd->done_task);
pm_runtime_disable(&pdev->dev);
dma_release_channel(dd->dma_lch);
- free_irq(dd->irq, dd);
- kfree(dd);
- dd = NULL;
return 0;
}
dev->req->base.complete(&dev->req->base, dev->error);
}
-void sahara_watchdog(unsigned long data)
+static void sahara_watchdog(unsigned long data)
{
struct sahara_dev *dev = (struct sahara_dev *)data;
unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
dev->hw_link[0] = dma_alloc_coherent(&pdev->dev,
SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
&dev->hw_phys_link[0], GFP_KERNEL);
- if (!dev->hw_link) {
+ if (!dev->hw_link[0]) {
dev_err(&pdev->dev, "Could not allocate hw links\n");
err = -ENOMEM;
goto err_link;
value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
- } while (eng_busy & (!icq_empty));
+ } while (eng_busy && !icq_empty);
aes_writel(dd, cmdq[i], TEGRA_AES_ICMDQUE_WR);
}
eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
dma_busy = value & TEGRA_AES_DMA_BUSY_FIELD;
- } while (eng_busy & (!icq_empty) & dma_busy);
+ } while (eng_busy && !icq_empty && dma_busy);
/* settable command to get key into internal registers */
value = CMD_SETTABLE << CMDQ_OPCODE_SHIFT |
value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
- } while (eng_busy & (!icq_empty));
+ } while (eng_busy && !icq_empty);
return 0;
}
* License terms: GNU General Public License (GPL) version 2
*/
+#define pr_fmt(fmt) "hashX hashX: " fmt
+
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/err.h>
#include "hash_alg.h"
-#define DEV_DBG_NAME "hashX hashX:"
-
static int hash_mode;
module_param(hash_mode, int, 0);
MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1");
/**
* Pre-calculated empty message digests.
*/
-static u8 zero_message_hash_sha1[SHA1_DIGEST_SIZE] = {
+static const u8 zero_message_hash_sha1[SHA1_DIGEST_SIZE] = {
0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
0xaf, 0xd8, 0x07, 0x09
};
-static u8 zero_message_hash_sha256[SHA256_DIGEST_SIZE] = {
+static const u8 zero_message_hash_sha256[SHA256_DIGEST_SIZE] = {
0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
};
/* HMAC-SHA1, no key */
-static u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = {
+static const u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = {
0xfb, 0xdb, 0x1d, 0x1b, 0x18, 0xaa, 0x6c, 0x08,
0x32, 0x4b, 0x7d, 0x64, 0xb7, 0x1f, 0xb7, 0x63,
0x70, 0x69, 0x0e, 0x1d
};
/* HMAC-SHA256, no key */
-static u8 zero_message_hmac_sha256[SHA256_DIGEST_SIZE] = {
+static const u8 zero_message_hmac_sha256[SHA256_DIGEST_SIZE] = {
0xb6, 0x13, 0x67, 0x9a, 0x08, 0x14, 0xd9, 0xec,
0x77, 0x2f, 0x95, 0xd7, 0x78, 0xc3, 0x5f, 0xc5,
0xff, 0x16, 0x97, 0xc4, 0x93, 0x71, 0x56, 0x53,
*
*/
static void hash_messagepad(struct hash_device_data *device_data,
- const u32 *message, u8 index_bytes);
+ const u32 *message, u8 index_bytes);
/**
* release_hash_device - Releases a previously allocated hash device.
}
static void hash_dma_setup_channel(struct hash_device_data *device_data,
- struct device *dev)
+ struct device *dev)
{
struct hash_platform_data *platform_data = dev->platform_data;
struct dma_slave_config conf = {
.dst_addr = device_data->phybase + HASH_DMA_FIFO,
.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
.dst_maxburst = 16,
- };
+ };
dma_cap_zero(device_data->dma.mask);
dma_cap_set(DMA_SLAVE, device_data->dma.mask);
device_data->dma.cfg_mem2hash = platform_data->mem_to_engine;
device_data->dma.chan_mem2hash =
dma_request_channel(device_data->dma.mask,
- platform_data->dma_filter,
- device_data->dma.cfg_mem2hash);
+ platform_data->dma_filter,
+ device_data->dma.cfg_mem2hash);
dmaengine_slave_config(device_data->dma.chan_mem2hash, &conf);
static void hash_dma_callback(void *data)
{
- struct hash_ctx *ctx = (struct hash_ctx *) data;
+ struct hash_ctx *ctx = data;
complete(&ctx->device->dma.complete);
}
static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg,
- int len, enum dma_data_direction direction)
+ int len, enum dma_data_direction direction)
{
struct dma_async_tx_descriptor *desc = NULL;
struct dma_chan *channel = NULL;
dma_cookie_t cookie;
if (direction != DMA_TO_DEVICE) {
- dev_err(ctx->device->dev, "[%s] Invalid DMA direction",
- __func__);
+ dev_err(ctx->device->dev, "%s: Invalid DMA direction\n",
+ __func__);
return -EFAULT;
}
direction);
if (!ctx->device->dma.sg_len) {
- dev_err(ctx->device->dev,
- "[%s]: Could not map the sg list (TO_DEVICE)",
- __func__);
+ dev_err(ctx->device->dev, "%s: Could not map the sg list (TO_DEVICE)\n",
+ __func__);
return -EFAULT;
}
- dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
- "(TO_DEVICE)", __func__);
+ dev_dbg(ctx->device->dev, "%s: Setting up DMA for buffer (TO_DEVICE)\n",
+ __func__);
desc = dmaengine_prep_slave_sg(channel,
ctx->device->dma.sg, ctx->device->dma.sg_len,
direction, DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
if (!desc) {
dev_err(ctx->device->dev,
- "[%s]: device_prep_slave_sg() failed!", __func__);
+ "%s: device_prep_slave_sg() failed!\n", __func__);
return -EFAULT;
}
chan = ctx->device->dma.chan_mem2hash;
dmaengine_device_control(chan, DMA_TERMINATE_ALL, 0);
dma_unmap_sg(chan->device->dev, ctx->device->dma.sg,
- ctx->device->dma.sg_len, DMA_TO_DEVICE);
-
+ ctx->device->dma.sg_len, DMA_TO_DEVICE);
}
static int hash_dma_write(struct hash_ctx *ctx,
- struct scatterlist *sg, int len)
+ struct scatterlist *sg, int len)
{
int error = hash_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
if (error) {
- dev_dbg(ctx->device->dev, "[%s]: hash_set_dma_transfer() "
- "failed", __func__);
+ dev_dbg(ctx->device->dev,
+ "%s: hash_set_dma_transfer() failed\n", __func__);
return error;
}
if (HASH_OPER_MODE_HASH == ctx->config.oper_mode) {
if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hash_sha1[0],
- SHA1_DIGEST_SIZE);
+ SHA1_DIGEST_SIZE);
*zero_hash_size = SHA1_DIGEST_SIZE;
*zero_digest = true;
} else if (HASH_ALGO_SHA256 ==
ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hash_sha256[0],
- SHA256_DIGEST_SIZE);
+ SHA256_DIGEST_SIZE);
*zero_hash_size = SHA256_DIGEST_SIZE;
*zero_digest = true;
} else {
- dev_err(device_data->dev, "[%s] "
- "Incorrect algorithm!"
- , __func__);
+ dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
+ __func__);
ret = -EINVAL;
goto out;
}
if (!ctx->keylen) {
if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hmac_sha1[0],
- SHA1_DIGEST_SIZE);
+ SHA1_DIGEST_SIZE);
*zero_hash_size = SHA1_DIGEST_SIZE;
*zero_digest = true;
} else if (HASH_ALGO_SHA256 == ctx->config.algorithm) {
memcpy(zero_hash, &zero_message_hmac_sha256[0],
- SHA256_DIGEST_SIZE);
+ SHA256_DIGEST_SIZE);
*zero_hash_size = SHA256_DIGEST_SIZE;
*zero_digest = true;
} else {
- dev_err(device_data->dev, "[%s] "
- "Incorrect algorithm!"
- , __func__);
+ dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
+ __func__);
ret = -EINVAL;
goto out;
}
} else {
- dev_dbg(device_data->dev, "[%s] Continue hash "
- "calculation, since hmac key available",
- __func__);
+ dev_dbg(device_data->dev,
+ "%s: Continue hash calculation, since hmac key available\n",
+ __func__);
}
}
out:
* This function request for disabling power (regulator) and clock,
* and could also save current hw state.
*/
-static int hash_disable_power(
- struct hash_device_data *device_data,
- bool save_device_state)
+static int hash_disable_power(struct hash_device_data *device_data,
+ bool save_device_state)
{
int ret = 0;
struct device *dev = device_data->dev;
clk_disable(device_data->clk);
ret = regulator_disable(device_data->regulator);
if (ret)
- dev_err(dev, "[%s] regulator_disable() failed!", __func__);
+ dev_err(dev, "%s: regulator_disable() failed!\n", __func__);
device_data->power_state = false;
* This function request for enabling power (regulator) and clock,
* and could also restore a previously saved hw state.
*/
-static int hash_enable_power(
- struct hash_device_data *device_data,
- bool restore_device_state)
+static int hash_enable_power(struct hash_device_data *device_data,
+ bool restore_device_state)
{
int ret = 0;
struct device *dev = device_data->dev;
if (!device_data->power_state) {
ret = regulator_enable(device_data->regulator);
if (ret) {
- dev_err(dev, "[%s]: regulator_enable() failed!",
- __func__);
+ dev_err(dev, "%s: regulator_enable() failed!\n",
+ __func__);
goto out;
}
ret = clk_enable(device_data->clk);
if (ret) {
- dev_err(dev, "[%s]: clk_enable() failed!",
- __func__);
+ dev_err(dev, "%s: clk_enable() failed!\n", __func__);
ret = regulator_disable(
device_data->regulator);
goto out;
if (device_data->restore_dev_state) {
if (restore_device_state) {
device_data->restore_dev_state = false;
- hash_resume_state(device_data,
- &device_data->state);
+ hash_resume_state(device_data, &device_data->state);
}
}
out:
* spec or due to a bug in the hw.
*/
static void hash_hw_write_key(struct hash_device_data *device_data,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
u32 word = 0;
int nwords = 1;
* calculation.
*/
static int init_hash_hw(struct hash_device_data *device_data,
- struct hash_ctx *ctx)
+ struct hash_ctx *ctx)
{
int ret = 0;
ret = hash_setconfiguration(device_data, &ctx->config);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_setconfiguration() "
- "failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_setconfiguration() failed!\n",
+ __func__);
return ret;
}
size -= sg->length;
/* hash_set_dma_transfer will align last nent */
- if ((aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE))
- || (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) &&
- size > 0))
+ if ((aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE)) ||
+ (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) && size > 0))
aligned_data = false;
sg = sg_next(sg);
if (req->nbytes < HASH_DMA_ALIGN_SIZE) {
req_ctx->dma_mode = false; /* Don't use DMA */
- pr_debug(DEV_DBG_NAME " [%s] DMA mode, but direct "
- "to CPU mode for data size < %d",
- __func__, HASH_DMA_ALIGN_SIZE);
+ pr_debug("%s: DMA mode, but direct to CPU mode for data size < %d\n",
+ __func__, HASH_DMA_ALIGN_SIZE);
} else {
if (req->nbytes >= HASH_DMA_PERFORMANCE_MIN_SIZE &&
- hash_dma_valid_data(req->src,
- req->nbytes)) {
+ hash_dma_valid_data(req->src, req->nbytes)) {
req_ctx->dma_mode = true;
} else {
req_ctx->dma_mode = false;
- pr_debug(DEV_DBG_NAME " [%s] DMA mode, but use"
- " CPU mode for datalength < %d"
- " or non-aligned data, except "
- "in last nent", __func__,
- HASH_DMA_PERFORMANCE_MIN_SIZE);
+ pr_debug("%s: DMA mode, but use CPU mode for datalength < %d or non-aligned data, except in last nent\n",
+ __func__,
+ HASH_DMA_PERFORMANCE_MIN_SIZE);
}
}
}
* the HASH hardware.
*
*/
-static void hash_processblock(
- struct hash_device_data *device_data,
- const u32 *message, int length)
+static void hash_processblock(struct hash_device_data *device_data,
+ const u32 *message, int length)
{
int len = length / HASH_BYTES_PER_WORD;
/*
*
*/
static void hash_messagepad(struct hash_device_data *device_data,
- const u32 *message, u8 index_bytes)
+ const u32 *message, u8 index_bytes)
{
int nwords = 1;
/* num_of_bytes == 0 => NBLW <- 0 (32 bits valid in DATAIN) */
HASH_SET_NBLW(index_bytes * 8);
- dev_dbg(device_data->dev, "[%s] DIN=0x%08x NBLW=%d", __func__,
- readl_relaxed(&device_data->base->din),
- (int)(readl_relaxed(&device_data->base->str) &
- HASH_STR_NBLW_MASK));
+ dev_dbg(device_data->dev, "%s: DIN=0x%08x NBLW=%lu\n",
+ __func__, readl_relaxed(&device_data->base->din),
+ readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
HASH_SET_DCAL;
- dev_dbg(device_data->dev, "[%s] after dcal -> DIN=0x%08x NBLW=%d",
- __func__, readl_relaxed(&device_data->base->din),
- (int)(readl_relaxed(&device_data->base->str) &
- HASH_STR_NBLW_MASK));
+ dev_dbg(device_data->dev, "%s: after dcal -> DIN=0x%08x NBLW=%lu\n",
+ __func__, readl_relaxed(&device_data->base->din),
+ readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
cpu_relax();
* @config: Pointer to a configuration structure.
*/
int hash_setconfiguration(struct hash_device_data *device_data,
- struct hash_config *config)
+ struct hash_config *config)
{
int ret = 0;
break;
default:
- dev_err(device_data->dev, "[%s] Incorrect algorithm.",
- __func__);
+ dev_err(device_data->dev, "%s: Incorrect algorithm\n",
+ __func__);
return -EPERM;
}
HASH_CLEAR_BITS(&device_data->base->cr,
HASH_CR_MODE_MASK);
else if (HASH_OPER_MODE_HMAC == config->oper_mode) {
- HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_MODE_MASK);
+ HASH_SET_BITS(&device_data->base->cr, HASH_CR_MODE_MASK);
if (device_data->current_ctx->keylen > HASH_BLOCK_SIZE) {
/* Truncate key to blocksize */
- dev_dbg(device_data->dev, "[%s] LKEY set", __func__);
+ dev_dbg(device_data->dev, "%s: LKEY set\n", __func__);
HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_LKEY_MASK);
+ HASH_CR_LKEY_MASK);
} else {
- dev_dbg(device_data->dev, "[%s] LKEY cleared",
- __func__);
+ dev_dbg(device_data->dev, "%s: LKEY cleared\n",
+ __func__);
HASH_CLEAR_BITS(&device_data->base->cr,
HASH_CR_LKEY_MASK);
}
} else { /* Wrong hash mode */
ret = -EPERM;
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
}
return ret;
}
}
static int hash_process_data(struct hash_device_data *device_data,
- struct hash_ctx *ctx, struct hash_req_ctx *req_ctx,
- int msg_length, u8 *data_buffer, u8 *buffer, u8 *index)
+ struct hash_ctx *ctx, struct hash_req_ctx *req_ctx,
+ int msg_length, u8 *data_buffer, u8 *buffer,
+ u8 *index)
{
int ret = 0;
u32 count;
msg_length = 0;
} else {
if (req_ctx->updated) {
-
ret = hash_resume_state(device_data,
&device_data->state);
memmove(req_ctx->state.buffer,
- device_data->state.buffer,
- HASH_BLOCK_SIZE / sizeof(u32));
+ device_data->state.buffer,
+ HASH_BLOCK_SIZE / sizeof(u32));
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "hash_resume_state()"
- " failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: hash_resume_state() failed!\n",
+ __func__);
goto out;
}
} else {
ret = init_hash_hw(device_data, ctx);
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "init_hash_hw()"
- " failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: init_hash_hw() failed!\n",
+ __func__);
goto out;
}
req_ctx->updated = 1;
* HW peripheral, otherwise we first copy data
* to a local buffer
*/
- if ((0 == (((u32)data_buffer) % 4))
- && (0 == *index))
+ if ((0 == (((u32)data_buffer) % 4)) &&
+ (0 == *index))
hash_processblock(device_data,
- (const u32 *)
- data_buffer, HASH_BLOCK_SIZE);
+ (const u32 *)data_buffer,
+ HASH_BLOCK_SIZE);
else {
- for (count = 0; count <
- (u32)(HASH_BLOCK_SIZE -
- *index);
- count++) {
+ for (count = 0;
+ count < (u32)(HASH_BLOCK_SIZE - *index);
+ count++) {
buffer[*index + count] =
*(data_buffer + count);
}
hash_processblock(device_data,
- (const u32 *)buffer,
- HASH_BLOCK_SIZE);
+ (const u32 *)buffer,
+ HASH_BLOCK_SIZE);
}
hash_incrementlength(req_ctx, HASH_BLOCK_SIZE);
data_buffer += (HASH_BLOCK_SIZE - *index);
&device_data->state);
memmove(device_data->state.buffer,
- req_ctx->state.buffer,
- HASH_BLOCK_SIZE / sizeof(u32));
+ req_ctx->state.buffer,
+ HASH_BLOCK_SIZE / sizeof(u32));
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "hash_save_state()"
- " failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_save_state() failed!\n",
+ __func__);
goto out;
}
}
if (ret)
return ret;
- dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx);
+ dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);
if (req_ctx->updated) {
ret = hash_resume_state(device_data, &device_data->state);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_resume_state() "
- "failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_resume_state() failed!\n",
+ __func__);
goto out;
}
-
}
if (!req_ctx->updated) {
ret = hash_setconfiguration(device_data, &ctx->config);
if (ret) {
- dev_err(device_data->dev, "[%s] "
- "hash_setconfiguration() failed!",
- __func__);
+ dev_err(device_data->dev,
+ "%s: hash_setconfiguration() failed!\n",
+ __func__);
goto out;
}
HASH_CR_DMAE_MASK);
} else {
HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_DMAE_MASK);
+ HASH_CR_DMAE_MASK);
HASH_SET_BITS(&device_data->base->cr,
- HASH_CR_PRIVN_MASK);
+ HASH_CR_PRIVN_MASK);
}
HASH_INITIALIZE;
/* Store the nents in the dma struct. */
ctx->device->dma.nents = hash_get_nents(req->src, req->nbytes, NULL);
if (!ctx->device->dma.nents) {
- dev_err(device_data->dev, "[%s] "
- "ctx->device->dma.nents = 0", __func__);
+ dev_err(device_data->dev, "%s: ctx->device->dma.nents = 0\n",
+ __func__);
ret = ctx->device->dma.nents;
goto out;
}
bytes_written = hash_dma_write(ctx, req->src, req->nbytes);
if (bytes_written != req->nbytes) {
- dev_err(device_data->dev, "[%s] "
- "hash_dma_write() failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_dma_write() failed!\n",
+ __func__);
ret = bytes_written;
goto out;
}
unsigned int keylen = ctx->keylen;
u8 *key = ctx->key;
- dev_dbg(device_data->dev, "[%s] keylen: %d", __func__,
- ctx->keylen);
+ dev_dbg(device_data->dev, "%s: keylen: %d\n",
+ __func__, ctx->keylen);
hash_hw_write_key(device_data, key, keylen);
}
if (ret)
return ret;
- dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx);
+ dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);
if (req_ctx->updated) {
ret = hash_resume_state(device_data, &device_data->state);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_resume_state() "
- "failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: hash_resume_state() failed!\n", __func__);
goto out;
}
} else if (req->nbytes == 0 && ctx->keylen == 0) {
ret = get_empty_message_digest(device_data, &zero_hash[0],
&zero_hash_size, &zero_digest);
if (!ret && likely(zero_hash_size == ctx->digestsize) &&
- zero_digest) {
+ zero_digest) {
memcpy(req->result, &zero_hash[0], ctx->digestsize);
goto out;
} else if (!ret && !zero_digest) {
- dev_dbg(device_data->dev, "[%s] HMAC zero msg with "
- "key, continue...", __func__);
+ dev_dbg(device_data->dev,
+ "%s: HMAC zero msg with key, continue...\n",
+ __func__);
} else {
- dev_err(device_data->dev, "[%s] ret=%d, or wrong "
- "digest size? %s", __func__, ret,
- (zero_hash_size == ctx->digestsize) ?
- "true" : "false");
+ dev_err(device_data->dev,
+ "%s: ret=%d, or wrong digest size? %s\n",
+ __func__, ret,
+ zero_hash_size == ctx->digestsize ?
+ "true" : "false");
/* Return error */
goto out;
}
} else if (req->nbytes == 0 && ctx->keylen > 0) {
- dev_err(device_data->dev, "[%s] Empty message with "
- "keylength > 0, NOT supported.", __func__);
+ dev_err(device_data->dev, "%s: Empty message with keylength > 0, NOT supported\n",
+ __func__);
goto out;
}
if (!req_ctx->updated) {
ret = init_hash_hw(device_data, ctx);
if (ret) {
- dev_err(device_data->dev, "[%s] init_hash_hw() "
- "failed!", __func__);
+ dev_err(device_data->dev,
+ "%s: init_hash_hw() failed!\n", __func__);
goto out;
}
}
unsigned int keylen = ctx->keylen;
u8 *key = ctx->key;
- dev_dbg(device_data->dev, "[%s] keylen: %d", __func__,
- ctx->keylen);
+ dev_dbg(device_data->dev, "%s: keylen: %d\n",
+ __func__, ctx->keylen);
hash_hw_write_key(device_data, key, keylen);
}
/* Check if ctx->state.length + msg_length
overflows */
if (msg_length > (req_ctx->state.length.low_word + msg_length) &&
- HASH_HIGH_WORD_MAX_VAL ==
- req_ctx->state.length.high_word) {
- pr_err(DEV_DBG_NAME " [%s] HASH_MSG_LENGTH_OVERFLOW!",
- __func__);
+ HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) {
+ pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__);
return -EPERM;
}
data_buffer, buffer, &index);
if (ret) {
- dev_err(device_data->dev, "[%s] hash_internal_hw_"
- "update() failed!", __func__);
+ dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n",
+ __func__);
goto out;
}
}
req_ctx->state.index = index;
- dev_dbg(device_data->dev, "[%s] indata length=%d, bin=%d))",
- __func__, req_ctx->state.index,
- req_ctx->state.bit_index);
+ dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n",
+ __func__, req_ctx->state.index, req_ctx->state.bit_index);
out:
release_hash_device(device_data);
* @device_state: The state to be restored in the hash hardware
*/
int hash_resume_state(struct hash_device_data *device_data,
- const struct hash_state *device_state)
+ const struct hash_state *device_state)
{
u32 temp_cr;
s32 count;
int hash_mode = HASH_OPER_MODE_HASH;
if (NULL == device_state) {
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
return -EPERM;
}
/* Check correctness of index and length members */
- if (device_state->index > HASH_BLOCK_SIZE
- || (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) {
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ if (device_state->index > HASH_BLOCK_SIZE ||
+ (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) {
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
return -EPERM;
}
break;
writel_relaxed(device_state->csr[count],
- &device_data->base->csrx[count]);
+ &device_data->base->csrx[count]);
}
writel_relaxed(device_state->csfull, &device_data->base->csfull);
* @device_state: The strucure where the hardware state should be saved.
*/
int hash_save_state(struct hash_device_data *device_data,
- struct hash_state *device_state)
+ struct hash_state *device_state)
{
u32 temp_cr;
u32 count;
int hash_mode = HASH_OPER_MODE_HASH;
if (NULL == device_state) {
- dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
+ __func__);
return -ENOTSUPP;
}
int hash_check_hw(struct hash_device_data *device_data)
{
/* Checking Peripheral Ids */
- if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0)
- && HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1)
- && HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2)
- && HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3)
- && HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0)
- && HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1)
- && HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2)
- && HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)
- ) {
+ if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0) &&
+ HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1) &&
+ HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2) &&
+ HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3) &&
+ HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0) &&
+ HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1) &&
+ HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2) &&
+ HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)) {
return 0;
}
- dev_err(device_data->dev, "[%s] HASH_UNSUPPORTED_HW!",
- __func__);
+ dev_err(device_data->dev, "%s: HASH_UNSUPPORTED_HW!\n", __func__);
return -ENOTSUPP;
}
* @algorithm: The algorithm in use.
*/
void hash_get_digest(struct hash_device_data *device_data,
- u8 *digest, int algorithm)
+ u8 *digest, int algorithm)
{
u32 temp_hx_val, count;
int loop_ctr;
if (algorithm != HASH_ALGO_SHA1 && algorithm != HASH_ALGO_SHA256) {
- dev_err(device_data->dev, "[%s] Incorrect algorithm %d",
- __func__, algorithm);
+ dev_err(device_data->dev, "%s: Incorrect algorithm %d\n",
+ __func__, algorithm);
return;
}
else
loop_ctr = SHA256_DIGEST_SIZE / sizeof(u32);
- dev_dbg(device_data->dev, "[%s] digest array:(0x%x)",
- __func__, (u32) digest);
+ dev_dbg(device_data->dev, "%s: digest array:(0x%x)\n",
+ __func__, (u32) digest);
/* Copy result into digest array */
for (count = 0; count < loop_ctr; count++) {
/* Skip update for DMA, all data will be passed to DMA in final */
if (ret) {
- pr_err(DEV_DBG_NAME " [%s] hash_hw_update() failed!",
- __func__);
+ pr_err("%s: hash_hw_update() failed!\n", __func__);
}
return ret;
int ret = 0;
struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
- pr_debug(DEV_DBG_NAME " [%s] data size: %d", __func__, req->nbytes);
+ pr_debug("%s: data size: %d\n", __func__, req->nbytes);
if ((hash_mode == HASH_MODE_DMA) && req_ctx->dma_mode)
ret = hash_dma_final(req);
ret = hash_hw_final(req);
if (ret) {
- pr_err(DEV_DBG_NAME " [%s] hash_hw/dma_final() failed",
- __func__);
+ pr_err("%s: hash_hw/dma_final() failed\n", __func__);
}
return ret;
}
static int hash_setkey(struct crypto_ahash *tfm,
- const u8 *key, unsigned int keylen, int alg)
+ const u8 *key, unsigned int keylen, int alg)
{
int ret = 0;
struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
*/
ctx->key = kmemdup(key, keylen, GFP_KERNEL);
if (!ctx->key) {
- pr_err(DEV_DBG_NAME " [%s] Failed to allocate ctx->key "
- "for %d\n", __func__, alg);
+ pr_err("%s: Failed to allocate ctx->key for %d\n",
+ __func__, alg);
return -ENOMEM;
}
ctx->keylen = keylen;
}
static int hmac_sha1_setkey(struct crypto_ahash *tfm,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA1);
}
static int hmac_sha256_setkey(struct crypto_ahash *tfm,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned int keylen)
{
return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA256);
}
hash);
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
- sizeof(struct hash_req_ctx));
+ sizeof(struct hash_req_ctx));
ctx->config.data_format = HASH_DATA_8_BITS;
ctx->config.algorithm = hash_alg->conf.algorithm;
static struct hash_algo_template hash_algs[] = {
{
- .conf.algorithm = HASH_ALGO_SHA1,
- .conf.oper_mode = HASH_OPER_MODE_HASH,
- .hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = ahash_sha1_digest,
- .halg.digestsize = SHA1_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "sha1",
- .cra_driver_name = "sha1-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
+ .conf.algorithm = HASH_ALGO_SHA1,
+ .conf.oper_mode = HASH_OPER_MODE_HASH,
+ .hash = {
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = ahash_sha1_digest,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
}
},
{
- .conf.algorithm = HASH_ALGO_SHA256,
- .conf.oper_mode = HASH_OPER_MODE_HASH,
- .hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = ahash_sha256_digest,
- .halg.digestsize = SHA256_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "sha256",
- .cra_driver_name = "sha256-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_type = &crypto_ahash_type,
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
- }
+ .conf.algorithm = HASH_ALGO_SHA256,
+ .conf.oper_mode = HASH_OPER_MODE_HASH,
+ .hash = {
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = ahash_sha256_digest,
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_type = &crypto_ahash_type,
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
-
+ }
},
{
- .conf.algorithm = HASH_ALGO_SHA1,
- .conf.oper_mode = HASH_OPER_MODE_HMAC,
+ .conf.algorithm = HASH_ALGO_SHA1,
+ .conf.oper_mode = HASH_OPER_MODE_HMAC,
.hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = hmac_sha1_digest,
- .setkey = hmac_sha1_setkey,
- .halg.digestsize = SHA1_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "hmac(sha1)",
- .cra_driver_name = "hmac-sha1-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_type = &crypto_ahash_type,
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
- }
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = hmac_sha1_digest,
+ .setkey = hmac_sha1_setkey,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "hmac-sha1-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_type = &crypto_ahash_type,
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
+ }
},
{
- .conf.algorithm = HASH_ALGO_SHA256,
- .conf.oper_mode = HASH_OPER_MODE_HMAC,
- .hash = {
- .init = hash_init,
- .update = ahash_update,
- .final = ahash_final,
- .digest = hmac_sha256_digest,
- .setkey = hmac_sha256_setkey,
- .halg.digestsize = SHA256_DIGEST_SIZE,
- .halg.statesize = sizeof(struct hash_ctx),
- .halg.base = {
- .cra_name = "hmac(sha256)",
- .cra_driver_name = "hmac-sha256-ux500",
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct hash_ctx),
- .cra_type = &crypto_ahash_type,
- .cra_init = hash_cra_init,
- .cra_module = THIS_MODULE,
- }
+ .conf.algorithm = HASH_ALGO_SHA256,
+ .conf.oper_mode = HASH_OPER_MODE_HMAC,
+ .hash = {
+ .init = hash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .digest = hmac_sha256_digest,
+ .setkey = hmac_sha256_setkey,
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct hash_ctx),
+ .halg.base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "hmac-sha256-ux500",
+ .cra_flags = (CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC),
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct hash_ctx),
+ .cra_type = &crypto_ahash_type,
+ .cra_init = hash_cra_init,
+ .cra_module = THIS_MODULE,
}
+ }
}
};
ret = crypto_register_ahash(&hash_algs[i].hash);
if (ret) {
count = i;
- dev_err(device_data->dev, "[%s] alg registration failed",
+ dev_err(device_data->dev, "%s: alg registration failed\n",
hash_algs[i].hash.halg.base.cra_driver_name);
goto unreg;
}
struct hash_device_data *device_data;
struct device *dev = &pdev->dev;
- device_data = kzalloc(sizeof(struct hash_device_data), GFP_ATOMIC);
+ device_data = kzalloc(sizeof(*device_data), GFP_ATOMIC);
if (!device_data) {
- dev_dbg(dev, "[%s] kzalloc() failed!", __func__);
ret = -ENOMEM;
goto out;
}
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res) {
- dev_dbg(dev, "[%s] platform_get_resource() failed!", __func__);
+ dev_dbg(dev, "%s: platform_get_resource() failed!\n", __func__);
ret = -ENODEV;
goto out_kfree;
}
res = request_mem_region(res->start, resource_size(res), pdev->name);
if (res == NULL) {
- dev_dbg(dev, "[%s] request_mem_region() failed!", __func__);
+ dev_dbg(dev, "%s: request_mem_region() failed!\n", __func__);
ret = -EBUSY;
goto out_kfree;
}
device_data->phybase = res->start;
device_data->base = ioremap(res->start, resource_size(res));
if (!device_data->base) {
- dev_err(dev, "[%s] ioremap() failed!",
- __func__);
+ dev_err(dev, "%s: ioremap() failed!\n", __func__);
ret = -ENOMEM;
goto out_free_mem;
}
/* Enable power for HASH1 hardware block */
device_data->regulator = regulator_get(dev, "v-ape");
if (IS_ERR(device_data->regulator)) {
- dev_err(dev, "[%s] regulator_get() failed!", __func__);
+ dev_err(dev, "%s: regulator_get() failed!\n", __func__);
ret = PTR_ERR(device_data->regulator);
device_data->regulator = NULL;
goto out_unmap;
/* Enable the clock for HASH1 hardware block */
device_data->clk = clk_get(dev, NULL);
if (IS_ERR(device_data->clk)) {
- dev_err(dev, "[%s] clk_get() failed!", __func__);
+ dev_err(dev, "%s: clk_get() failed!\n", __func__);
ret = PTR_ERR(device_data->clk);
goto out_regulator;
}
ret = clk_prepare(device_data->clk);
if (ret) {
- dev_err(dev, "[%s] clk_prepare() failed!", __func__);
+ dev_err(dev, "%s: clk_prepare() failed!\n", __func__);
goto out_clk;
}
/* Enable device power (and clock) */
ret = hash_enable_power(device_data, false);
if (ret) {
- dev_err(dev, "[%s]: hash_enable_power() failed!", __func__);
+ dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
goto out_clk_unprepare;
}
ret = hash_check_hw(device_data);
if (ret) {
- dev_err(dev, "[%s] hash_check_hw() failed!", __func__);
+ dev_err(dev, "%s: hash_check_hw() failed!\n", __func__);
goto out_power;
}
ret = ahash_algs_register_all(device_data);
if (ret) {
- dev_err(dev, "[%s] ahash_algs_register_all() "
- "failed!", __func__);
+ dev_err(dev, "%s: ahash_algs_register_all() failed!\n",
+ __func__);
goto out_power;
}
device_data = platform_get_drvdata(pdev);
if (!device_data) {
- dev_err(dev, "[%s]: platform_get_drvdata() failed!",
- __func__);
+ dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
return -ENOMEM;
}
ahash_algs_unregister_all(device_data);
if (hash_disable_power(device_data, false))
- dev_err(dev, "[%s]: hash_disable_power() failed",
+ dev_err(dev, "%s: hash_disable_power() failed\n",
__func__);
clk_unprepare(device_data->clk);
device_data = platform_get_drvdata(pdev);
if (!device_data) {
- dev_err(&pdev->dev, "[%s] platform_get_drvdata() failed!",
- __func__);
+ dev_err(&pdev->dev, "%s: platform_get_drvdata() failed!\n",
+ __func__);
return;
}
/* current_ctx allocates a device, NULL = unallocated */
if (!device_data->current_ctx) {
if (down_trylock(&driver_data.device_allocation))
- dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
- "Shutting down anyway...", __func__);
+ dev_dbg(&pdev->dev, "%s: Cryp still in use! Shutting down anyway...\n",
+ __func__);
/**
* (Allocate the device)
* Need to set this to non-null (dummy) value,
release_mem_region(res->start, resource_size(res));
if (hash_disable_power(device_data, false))
- dev_err(&pdev->dev, "[%s] hash_disable_power() failed",
- __func__);
+ dev_err(&pdev->dev, "%s: hash_disable_power() failed\n",
+ __func__);
}
/**
device_data = dev_get_drvdata(dev);
if (!device_data) {
- dev_err(dev, "[%s] platform_get_drvdata() failed!", __func__);
+ dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
return -ENOMEM;
}
if (device_data->current_ctx == ++temp_ctx) {
if (down_interruptible(&driver_data.device_allocation))
- dev_dbg(dev, "[%s]: down_interruptible() failed",
+ dev_dbg(dev, "%s: down_interruptible() failed\n",
__func__);
ret = hash_disable_power(device_data, false);
- } else
+ } else {
ret = hash_disable_power(device_data, true);
+ }
if (ret)
- dev_err(dev, "[%s]: hash_disable_power()", __func__);
+ dev_err(dev, "%s: hash_disable_power()\n", __func__);
return ret;
}
device_data = dev_get_drvdata(dev);
if (!device_data) {
- dev_err(dev, "[%s] platform_get_drvdata() failed!", __func__);
+ dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
return -ENOMEM;
}
ret = hash_enable_power(device_data, true);
if (ret)
- dev_err(dev, "[%s]: hash_enable_power() failed!", __func__);
+ dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
return ret;
}
static SIMPLE_DEV_PM_OPS(ux500_hash_pm, ux500_hash_suspend, ux500_hash_resume);
static const struct of_device_id ux500_hash_match[] = {
- { .compatible = "stericsson,ux500-hash" },
- { },
+ { .compatible = "stericsson,ux500-hash" },
+ { },
};
static struct platform_driver hash_driver = {
void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg,
unsigned int start, unsigned int nbytes, int out);
+int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes);
+
#endif /* _CRYPTO_SCATTERWALK_H */
#include <linux/types.h>
+#define CRC_T10DIF_DIGEST_SIZE 2
+#define CRC_T10DIF_BLOCK_SIZE 1
+
+__u16 crc_t10dif_generic(__u16 crc, const unsigned char *buffer, size_t len);
__u16 crc_t10dif(unsigned char const *, size_t);
#endif
switch (action) {
case CPU_ONLINE:
case CPU_ONLINE_FROZEN:
+ case CPU_DOWN_FAILED:
+ case CPU_DOWN_FAILED_FROZEN:
if (!pinst_has_cpu(pinst, cpu))
break;
mutex_lock(&pinst->lock);
case CPU_DOWN_PREPARE:
case CPU_DOWN_PREPARE_FROZEN:
+ case CPU_UP_CANCELED:
+ case CPU_UP_CANCELED_FROZEN:
if (!pinst_has_cpu(pinst, cpu))
break;
mutex_lock(&pinst->lock);
if (err)
return notifier_from_errno(err);
break;
-
- case CPU_UP_CANCELED:
- case CPU_UP_CANCELED_FROZEN:
- if (!pinst_has_cpu(pinst, cpu))
- break;
- mutex_lock(&pinst->lock);
- __padata_remove_cpu(pinst, cpu);
- mutex_unlock(&pinst->lock);
-
- case CPU_DOWN_FAILED:
- case CPU_DOWN_FAILED_FROZEN:
- if (!pinst_has_cpu(pinst, cpu))
- break;
- mutex_lock(&pinst->lock);
- __padata_add_cpu(pinst, cpu);
- mutex_unlock(&pinst->lock);
}
return NOTIFY_OK;
pinst->flags = 0;
-#ifdef CONFIG_HOTPLUG_CPU
- pinst->cpu_notifier.notifier_call = padata_cpu_callback;
- pinst->cpu_notifier.priority = 0;
- register_hotcpu_notifier(&pinst->cpu_notifier);
-#endif
-
put_online_cpus();
BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier);
kobject_init(&pinst->kobj, &padata_attr_type);
mutex_init(&pinst->lock);
+#ifdef CONFIG_HOTPLUG_CPU
+ pinst->cpu_notifier.notifier_call = padata_cpu_callback;
+ pinst->cpu_notifier.priority = 0;
+ register_hotcpu_notifier(&pinst->cpu_notifier);
+#endif
+
return pinst;
err_free_masks:
config CRC_T10DIF
tristate "CRC calculation for the T10 Data Integrity Field"
+ select CRYPTO
+ select CRYPTO_CRCT10DIF
help
This option is only needed if a module that's not in the
kernel tree needs to calculate CRC checks for use with the
#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-t10dif.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <crypto/hash.h>
-/* Table generated using the following polynomium:
- * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1
- * gt: 0x8bb7
- */
-static const __u16 t10_dif_crc_table[256] = {
- 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
- 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
- 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
- 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
- 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
- 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
- 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
- 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
- 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
- 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
- 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
- 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
- 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
- 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
- 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
- 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
- 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
- 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
- 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
- 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
- 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
- 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
- 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
- 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
- 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
- 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
- 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
- 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
- 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
- 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
- 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
- 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
-};
+static struct crypto_shash *crct10dif_tfm;
__u16 crc_t10dif(const unsigned char *buffer, size_t len)
{
- __u16 crc = 0;
- unsigned int i;
+ struct {
+ struct shash_desc shash;
+ char ctx[2];
+ } desc;
+ int err;
+
+ desc.shash.tfm = crct10dif_tfm;
+ desc.shash.flags = 0;
+ *(__u16 *)desc.ctx = 0;
- for (i = 0 ; i < len ; i++)
- crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff];
+ err = crypto_shash_update(&desc.shash, buffer, len);
+ BUG_ON(err);
- return crc;
+ return *(__u16 *)desc.ctx;
}
EXPORT_SYMBOL(crc_t10dif);
+static int __init crc_t10dif_mod_init(void)
+{
+ crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0);
+ return PTR_RET(crct10dif_tfm);
+}
+
+static void __exit crc_t10dif_mod_fini(void)
+{
+ crypto_free_shash(crct10dif_tfm);
+}
+
+module_init(crc_t10dif_mod_init);
+module_exit(crc_t10dif_mod_fini);
+
MODULE_DESCRIPTION("T10 DIF CRC calculation");
MODULE_LICENSE("GPL");
+MODULE_SOFTDEP("pre: crct10dif");
projects / cascardo / linux.git / commitdiff