1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. 6 * 7 * Author: Gary R Hook <gary.hook@amd.com> 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/kthread.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/interrupt.h> 14 #include <linux/compiler.h> 15 #include <linux/ccp.h> 16 17 #include "ccp-dev.h" 18 19 /* Allocate the requested number of contiguous LSB slots 20 * from the LSB bitmap. Look in the private range for this 21 * queue first; failing that, check the public area. 22 * If no space is available, wait around. 23 * Return: first slot number 24 */ ccp_lsb_alloc(struct ccp_cmd_queue * cmd_q,unsigned int count)25 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) 26 { 27 struct ccp_device *ccp; 28 int start; 29 30 /* First look at the map for the queue */ 31 if (cmd_q->lsb >= 0) { 32 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap, 33 LSB_SIZE, 34 0, count, 0); 35 if (start < LSB_SIZE) { 36 bitmap_set(cmd_q->lsbmap, start, count); 37 return start + cmd_q->lsb * LSB_SIZE; 38 } 39 } 40 41 /* No joy; try to get an entry from the shared blocks */ 42 ccp = cmd_q->ccp; 43 for (;;) { 44 mutex_lock(&ccp->sb_mutex); 45 46 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap, 47 MAX_LSB_CNT * LSB_SIZE, 48 0, 49 count, 0); 50 if (start <= MAX_LSB_CNT * LSB_SIZE) { 51 bitmap_set(ccp->lsbmap, start, count); 52 53 mutex_unlock(&ccp->sb_mutex); 54 return start; 55 } 56 57 ccp->sb_avail = 0; 58 59 mutex_unlock(&ccp->sb_mutex); 60 61 /* Wait for KSB entries to become available */ 62 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) 63 return 0; 64 } 65 } 66 67 /* Free a number of LSB slots from the bitmap, starting at 68 * the indicated starting slot number. 69 */ ccp_lsb_free(struct ccp_cmd_queue * cmd_q,unsigned int start,unsigned int count)70 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, 71 unsigned int count) 72 { 73 if (!start) 74 return; 75 76 if (cmd_q->lsb == start) { 77 /* An entry from the private LSB */ 78 bitmap_clear(cmd_q->lsbmap, start, count); 79 } else { 80 /* From the shared LSBs */ 81 struct ccp_device *ccp = cmd_q->ccp; 82 83 mutex_lock(&ccp->sb_mutex); 84 bitmap_clear(ccp->lsbmap, start, count); 85 ccp->sb_avail = 1; 86 mutex_unlock(&ccp->sb_mutex); 87 wake_up_interruptible_all(&ccp->sb_queue); 88 } 89 } 90 91 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ 92 union ccp_function { 93 struct { 94 u16 size:7; 95 u16 encrypt:1; 96 u16 mode:5; 97 u16 type:2; 98 } aes; 99 struct { 100 u16 size:7; 101 u16 encrypt:1; 102 u16 rsvd:5; 103 u16 type:2; 104 } aes_xts; 105 struct { 106 u16 size:7; 107 u16 encrypt:1; 108 u16 mode:5; 109 u16 type:2; 110 } des3; 111 struct { 112 u16 rsvd1:10; 113 u16 type:4; 114 u16 rsvd2:1; 115 } sha; 116 struct { 117 u16 mode:3; 118 u16 size:12; 119 } rsa; 120 struct { 121 u16 byteswap:2; 122 u16 bitwise:3; 123 u16 reflect:2; 124 u16 rsvd:8; 125 } pt; 126 struct { 127 u16 rsvd:13; 128 } zlib; 129 struct { 130 u16 size:10; 131 u16 type:2; 132 u16 mode:3; 133 } ecc; 134 u16 raw; 135 }; 136 137 #define CCP_AES_SIZE(p) ((p)->aes.size) 138 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) 139 #define CCP_AES_MODE(p) ((p)->aes.mode) 140 #define CCP_AES_TYPE(p) ((p)->aes.type) 141 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) 142 #define CCP_XTS_TYPE(p) ((p)->aes_xts.type) 143 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) 144 #define CCP_DES3_SIZE(p) ((p)->des3.size) 145 #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) 146 #define CCP_DES3_MODE(p) ((p)->des3.mode) 147 #define CCP_DES3_TYPE(p) ((p)->des3.type) 148 #define CCP_SHA_TYPE(p) ((p)->sha.type) 149 #define CCP_RSA_SIZE(p) ((p)->rsa.size) 150 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) 151 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise) 152 #define CCP_ECC_MODE(p) ((p)->ecc.mode) 153 #define CCP_ECC_AFFINE(p) ((p)->ecc.one) 154 155 /* Word 0 */ 156 #define CCP5_CMD_DW0(p) ((p)->dw0) 157 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) 158 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) 159 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) 160 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) 161 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) 162 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) 163 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) 164 165 /* Word 1 */ 166 #define CCP5_CMD_DW1(p) ((p)->length) 167 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) 168 169 /* Word 2 */ 170 #define CCP5_CMD_DW2(p) ((p)->src_lo) 171 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) 172 173 /* Word 3 */ 174 #define CCP5_CMD_DW3(p) ((p)->dw3) 175 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) 176 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) 177 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) 178 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) 179 180 /* Words 4/5 */ 181 #define CCP5_CMD_DW4(p) ((p)->dw4) 182 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) 183 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) 184 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) 185 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) 186 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) 187 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) 188 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) 189 190 /* Word 6/7 */ 191 #define CCP5_CMD_DW6(p) ((p)->key_lo) 192 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) 193 #define CCP5_CMD_DW7(p) ((p)->dw7) 194 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) 195 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) 196 low_address(unsigned long addr)197 static inline u32 low_address(unsigned long addr) 198 { 199 return (u64)addr & 0x0ffffffff; 200 } 201 high_address(unsigned long addr)202 static inline u32 high_address(unsigned long addr) 203 { 204 return ((u64)addr >> 32) & 0x00000ffff; 205 } 206 ccp5_get_free_slots(struct ccp_cmd_queue * cmd_q)207 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) 208 { 209 unsigned int head_idx, n; 210 u32 head_lo, queue_start; 211 212 queue_start = low_address(cmd_q->qdma_tail); 213 head_lo = ioread32(cmd_q->reg_head_lo); 214 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); 215 216 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; 217 218 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ 219 } 220 ccp5_do_cmd(struct ccp5_desc * desc,struct ccp_cmd_queue * cmd_q)221 static int ccp5_do_cmd(struct ccp5_desc *desc, 222 struct ccp_cmd_queue *cmd_q) 223 { 224 __le32 *mP; 225 u32 *dP; 226 u32 tail; 227 int i; 228 int ret = 0; 229 230 cmd_q->total_ops++; 231 232 if (CCP5_CMD_SOC(desc)) { 233 CCP5_CMD_IOC(desc) = 1; 234 CCP5_CMD_SOC(desc) = 0; 235 } 236 mutex_lock(&cmd_q->q_mutex); 237 238 mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx]; 239 dP = (u32 *)desc; 240 for (i = 0; i < 8; i++) 241 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ 242 243 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 244 245 /* The data used by this command must be flushed to memory */ 246 wmb(); 247 248 /* Write the new tail address back to the queue register */ 249 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 250 iowrite32(tail, cmd_q->reg_tail_lo); 251 252 /* Turn the queue back on using our cached control register */ 253 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); 254 mutex_unlock(&cmd_q->q_mutex); 255 256 if (CCP5_CMD_IOC(desc)) { 257 /* Wait for the job to complete */ 258 ret = wait_event_interruptible(cmd_q->int_queue, 259 cmd_q->int_rcvd); 260 if (ret || cmd_q->cmd_error) { 261 /* Log the error and flush the queue by 262 * moving the head pointer 263 */ 264 if (cmd_q->cmd_error) 265 ccp_log_error(cmd_q->ccp, 266 cmd_q->cmd_error); 267 iowrite32(tail, cmd_q->reg_head_lo); 268 if (!ret) 269 ret = -EIO; 270 } 271 cmd_q->int_rcvd = 0; 272 } 273 274 return ret; 275 } 276 ccp5_perform_aes(struct ccp_op * op)277 static int ccp5_perform_aes(struct ccp_op *op) 278 { 279 struct ccp5_desc desc; 280 union ccp_function function; 281 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 282 283 op->cmd_q->total_aes_ops++; 284 285 /* Zero out all the fields of the command desc */ 286 memset(&desc, 0, Q_DESC_SIZE); 287 288 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; 289 290 CCP5_CMD_SOC(&desc) = op->soc; 291 CCP5_CMD_IOC(&desc) = 1; 292 CCP5_CMD_INIT(&desc) = op->init; 293 CCP5_CMD_EOM(&desc) = op->eom; 294 CCP5_CMD_PROT(&desc) = 0; 295 296 function.raw = 0; 297 CCP_AES_ENCRYPT(&function) = op->u.aes.action; 298 CCP_AES_MODE(&function) = op->u.aes.mode; 299 CCP_AES_TYPE(&function) = op->u.aes.type; 300 CCP_AES_SIZE(&function) = op->u.aes.size; 301 302 CCP5_CMD_FUNCTION(&desc) = function.raw; 303 304 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 305 306 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 307 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 308 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 309 310 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 311 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 312 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 313 314 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 315 CCP5_CMD_KEY_HI(&desc) = 0; 316 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 317 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 318 319 return ccp5_do_cmd(&desc, op->cmd_q); 320 } 321 ccp5_perform_xts_aes(struct ccp_op * op)322 static int ccp5_perform_xts_aes(struct ccp_op *op) 323 { 324 struct ccp5_desc desc; 325 union ccp_function function; 326 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 327 328 op->cmd_q->total_xts_aes_ops++; 329 330 /* Zero out all the fields of the command desc */ 331 memset(&desc, 0, Q_DESC_SIZE); 332 333 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; 334 335 CCP5_CMD_SOC(&desc) = op->soc; 336 CCP5_CMD_IOC(&desc) = 1; 337 CCP5_CMD_INIT(&desc) = op->init; 338 CCP5_CMD_EOM(&desc) = op->eom; 339 CCP5_CMD_PROT(&desc) = 0; 340 341 function.raw = 0; 342 CCP_XTS_TYPE(&function) = op->u.xts.type; 343 CCP_XTS_ENCRYPT(&function) = op->u.xts.action; 344 CCP_XTS_SIZE(&function) = op->u.xts.unit_size; 345 CCP5_CMD_FUNCTION(&desc) = function.raw; 346 347 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 348 349 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 350 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 351 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 352 353 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 354 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 355 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 356 357 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 358 CCP5_CMD_KEY_HI(&desc) = 0; 359 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 360 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 361 362 return ccp5_do_cmd(&desc, op->cmd_q); 363 } 364 ccp5_perform_sha(struct ccp_op * op)365 static int ccp5_perform_sha(struct ccp_op *op) 366 { 367 struct ccp5_desc desc; 368 union ccp_function function; 369 370 op->cmd_q->total_sha_ops++; 371 372 /* Zero out all the fields of the command desc */ 373 memset(&desc, 0, Q_DESC_SIZE); 374 375 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; 376 377 CCP5_CMD_SOC(&desc) = op->soc; 378 CCP5_CMD_IOC(&desc) = 1; 379 CCP5_CMD_INIT(&desc) = 1; 380 CCP5_CMD_EOM(&desc) = op->eom; 381 CCP5_CMD_PROT(&desc) = 0; 382 383 function.raw = 0; 384 CCP_SHA_TYPE(&function) = op->u.sha.type; 385 CCP5_CMD_FUNCTION(&desc) = function.raw; 386 387 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 388 389 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 390 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 391 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 392 393 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 394 395 if (op->eom) { 396 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); 397 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); 398 } else { 399 CCP5_CMD_SHA_LO(&desc) = 0; 400 CCP5_CMD_SHA_HI(&desc) = 0; 401 } 402 403 return ccp5_do_cmd(&desc, op->cmd_q); 404 } 405 ccp5_perform_des3(struct ccp_op * op)406 static int ccp5_perform_des3(struct ccp_op *op) 407 { 408 struct ccp5_desc desc; 409 union ccp_function function; 410 u32 key_addr = op->sb_key * LSB_ITEM_SIZE; 411 412 op->cmd_q->total_3des_ops++; 413 414 /* Zero out all the fields of the command desc */ 415 memset(&desc, 0, sizeof(struct ccp5_desc)); 416 417 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; 418 419 CCP5_CMD_SOC(&desc) = op->soc; 420 CCP5_CMD_IOC(&desc) = 1; 421 CCP5_CMD_INIT(&desc) = op->init; 422 CCP5_CMD_EOM(&desc) = op->eom; 423 CCP5_CMD_PROT(&desc) = 0; 424 425 function.raw = 0; 426 CCP_DES3_ENCRYPT(&function) = op->u.des3.action; 427 CCP_DES3_MODE(&function) = op->u.des3.mode; 428 CCP_DES3_TYPE(&function) = op->u.des3.type; 429 CCP5_CMD_FUNCTION(&desc) = function.raw; 430 431 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 432 433 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 434 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 435 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 436 437 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 438 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 439 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 440 441 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); 442 CCP5_CMD_KEY_HI(&desc) = 0; 443 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; 444 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; 445 446 return ccp5_do_cmd(&desc, op->cmd_q); 447 } 448 ccp5_perform_rsa(struct ccp_op * op)449 static int ccp5_perform_rsa(struct ccp_op *op) 450 { 451 struct ccp5_desc desc; 452 union ccp_function function; 453 454 op->cmd_q->total_rsa_ops++; 455 456 /* Zero out all the fields of the command desc */ 457 memset(&desc, 0, Q_DESC_SIZE); 458 459 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; 460 461 CCP5_CMD_SOC(&desc) = op->soc; 462 CCP5_CMD_IOC(&desc) = 1; 463 CCP5_CMD_INIT(&desc) = 0; 464 CCP5_CMD_EOM(&desc) = 1; 465 CCP5_CMD_PROT(&desc) = 0; 466 467 function.raw = 0; 468 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3; 469 CCP5_CMD_FUNCTION(&desc) = function.raw; 470 471 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; 472 473 /* Source is from external memory */ 474 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 475 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 476 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 477 478 /* Destination is in external memory */ 479 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 480 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 481 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 482 483 /* Key (Exponent) is in external memory */ 484 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma); 485 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma); 486 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 487 488 return ccp5_do_cmd(&desc, op->cmd_q); 489 } 490 ccp5_perform_passthru(struct ccp_op * op)491 static int ccp5_perform_passthru(struct ccp_op *op) 492 { 493 struct ccp5_desc desc; 494 union ccp_function function; 495 struct ccp_dma_info *saddr = &op->src.u.dma; 496 struct ccp_dma_info *daddr = &op->dst.u.dma; 497 498 499 op->cmd_q->total_pt_ops++; 500 501 memset(&desc, 0, Q_DESC_SIZE); 502 503 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; 504 505 CCP5_CMD_SOC(&desc) = 0; 506 CCP5_CMD_IOC(&desc) = 1; 507 CCP5_CMD_INIT(&desc) = 0; 508 CCP5_CMD_EOM(&desc) = op->eom; 509 CCP5_CMD_PROT(&desc) = 0; 510 511 function.raw = 0; 512 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; 513 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; 514 CCP5_CMD_FUNCTION(&desc) = function.raw; 515 516 /* Length of source data is always 256 bytes */ 517 if (op->src.type == CCP_MEMTYPE_SYSTEM) 518 CCP5_CMD_LEN(&desc) = saddr->length; 519 else 520 CCP5_CMD_LEN(&desc) = daddr->length; 521 522 if (op->src.type == CCP_MEMTYPE_SYSTEM) { 523 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 524 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 525 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 526 527 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 528 CCP5_CMD_LSB_ID(&desc) = op->sb_key; 529 } else { 530 u32 key_addr = op->src.u.sb * CCP_SB_BYTES; 531 532 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); 533 CCP5_CMD_SRC_HI(&desc) = 0; 534 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; 535 } 536 537 if (op->dst.type == CCP_MEMTYPE_SYSTEM) { 538 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 539 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 540 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 541 } else { 542 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; 543 544 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); 545 CCP5_CMD_DST_HI(&desc) = 0; 546 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; 547 } 548 549 return ccp5_do_cmd(&desc, op->cmd_q); 550 } 551 ccp5_perform_ecc(struct ccp_op * op)552 static int ccp5_perform_ecc(struct ccp_op *op) 553 { 554 struct ccp5_desc desc; 555 union ccp_function function; 556 557 op->cmd_q->total_ecc_ops++; 558 559 /* Zero out all the fields of the command desc */ 560 memset(&desc, 0, Q_DESC_SIZE); 561 562 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; 563 564 CCP5_CMD_SOC(&desc) = 0; 565 CCP5_CMD_IOC(&desc) = 1; 566 CCP5_CMD_INIT(&desc) = 0; 567 CCP5_CMD_EOM(&desc) = 1; 568 CCP5_CMD_PROT(&desc) = 0; 569 570 function.raw = 0; 571 function.ecc.mode = op->u.ecc.function; 572 CCP5_CMD_FUNCTION(&desc) = function.raw; 573 574 CCP5_CMD_LEN(&desc) = op->src.u.dma.length; 575 576 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma); 577 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma); 578 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 579 580 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma); 581 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); 582 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; 583 584 return ccp5_do_cmd(&desc, op->cmd_q); 585 } 586 ccp_find_lsb_regions(struct ccp_cmd_queue * cmd_q,u64 status)587 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) 588 { 589 int q_mask = 1 << cmd_q->id; 590 int queues = 0; 591 int j; 592 593 /* Build a bit mask to know which LSBs this queue has access to. 594 * Don't bother with segment 0 as it has special privileges. 595 */ 596 for (j = 1; j < MAX_LSB_CNT; j++) { 597 if (status & q_mask) 598 bitmap_set(cmd_q->lsbmask, j, 1); 599 status >>= LSB_REGION_WIDTH; 600 } 601 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 602 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n", 603 cmd_q->id, queues); 604 605 return queues ? 0 : -EINVAL; 606 } 607 ccp_find_and_assign_lsb_to_q(struct ccp_device * ccp,int lsb_cnt,int n_lsbs,unsigned long * lsb_pub)608 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, 609 int lsb_cnt, int n_lsbs, 610 unsigned long *lsb_pub) 611 { 612 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 613 int bitno; 614 int qlsb_wgt; 615 int i; 616 617 /* For each queue: 618 * If the count of potential LSBs available to a queue matches the 619 * ordinal given to us in lsb_cnt: 620 * Copy the mask of possible LSBs for this queue into "qlsb"; 621 * For each bit in qlsb, see if the corresponding bit in the 622 * aggregation mask is set; if so, we have a match. 623 * If we have a match, clear the bit in the aggregation to 624 * mark it as no longer available. 625 * If there is no match, clear the bit in qlsb and keep looking. 626 */ 627 for (i = 0; i < ccp->cmd_q_count; i++) { 628 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 629 630 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT); 631 632 if (qlsb_wgt == lsb_cnt) { 633 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT); 634 635 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 636 while (bitno < MAX_LSB_CNT) { 637 if (test_bit(bitno, lsb_pub)) { 638 /* We found an available LSB 639 * that this queue can access 640 */ 641 cmd_q->lsb = bitno; 642 bitmap_clear(lsb_pub, bitno, 1); 643 dev_dbg(ccp->dev, 644 "Queue %d gets LSB %d\n", 645 i, bitno); 646 break; 647 } 648 bitmap_clear(qlsb, bitno, 1); 649 bitno = find_first_bit(qlsb, MAX_LSB_CNT); 650 } 651 if (bitno >= MAX_LSB_CNT) 652 return -EINVAL; 653 n_lsbs--; 654 } 655 } 656 return n_lsbs; 657 } 658 659 /* For each queue, from the most- to least-constrained: 660 * find an LSB that can be assigned to the queue. If there are N queues that 661 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a 662 * dedicated LSB. Remaining LSB regions become a shared resource. 663 * If we have fewer LSBs than queues, all LSB regions become shared resources. 664 */ ccp_assign_lsbs(struct ccp_device * ccp)665 static int ccp_assign_lsbs(struct ccp_device *ccp) 666 { 667 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); 668 DECLARE_BITMAP(qlsb, MAX_LSB_CNT); 669 int n_lsbs = 0; 670 int bitno; 671 int i, lsb_cnt; 672 int rc = 0; 673 674 bitmap_zero(lsb_pub, MAX_LSB_CNT); 675 676 /* Create an aggregate bitmap to get a total count of available LSBs */ 677 for (i = 0; i < ccp->cmd_q_count; i++) 678 bitmap_or(lsb_pub, 679 lsb_pub, ccp->cmd_q[i].lsbmask, 680 MAX_LSB_CNT); 681 682 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT); 683 684 if (n_lsbs >= ccp->cmd_q_count) { 685 /* We have enough LSBS to give every queue a private LSB. 686 * Brute force search to start with the queues that are more 687 * constrained in LSB choice. When an LSB is privately 688 * assigned, it is removed from the public mask. 689 * This is an ugly N squared algorithm with some optimization. 690 */ 691 for (lsb_cnt = 1; 692 n_lsbs && (lsb_cnt <= MAX_LSB_CNT); 693 lsb_cnt++) { 694 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, 695 lsb_pub); 696 if (rc < 0) 697 return -EINVAL; 698 n_lsbs = rc; 699 } 700 } 701 702 rc = 0; 703 /* What's left of the LSBs, according to the public mask, now become 704 * shared. Any zero bits in the lsb_pub mask represent an LSB region 705 * that can't be used as a shared resource, so mark the LSB slots for 706 * them as "in use". 707 */ 708 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT); 709 710 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 711 while (bitno < MAX_LSB_CNT) { 712 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE); 713 bitmap_set(qlsb, bitno, 1); 714 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT); 715 } 716 717 return rc; 718 } 719 ccp5_disable_queue_interrupts(struct ccp_device * ccp)720 static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) 721 { 722 unsigned int i; 723 724 for (i = 0; i < ccp->cmd_q_count; i++) 725 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); 726 } 727 ccp5_enable_queue_interrupts(struct ccp_device * ccp)728 static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) 729 { 730 unsigned int i; 731 732 for (i = 0; i < ccp->cmd_q_count; i++) 733 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); 734 } 735 ccp5_irq_bh(unsigned long data)736 static void ccp5_irq_bh(unsigned long data) 737 { 738 struct ccp_device *ccp = (struct ccp_device *)data; 739 u32 status; 740 unsigned int i; 741 742 for (i = 0; i < ccp->cmd_q_count; i++) { 743 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; 744 745 status = ioread32(cmd_q->reg_interrupt_status); 746 747 if (status) { 748 cmd_q->int_status = status; 749 cmd_q->q_status = ioread32(cmd_q->reg_status); 750 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); 751 752 /* On error, only save the first error value */ 753 if ((status & INT_ERROR) && !cmd_q->cmd_error) 754 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 755 756 cmd_q->int_rcvd = 1; 757 758 /* Acknowledge the interrupt and wake the kthread */ 759 iowrite32(status, cmd_q->reg_interrupt_status); 760 wake_up_interruptible(&cmd_q->int_queue); 761 } 762 } 763 ccp5_enable_queue_interrupts(ccp); 764 } 765 ccp5_irq_handler(int irq,void * data)766 static irqreturn_t ccp5_irq_handler(int irq, void *data) 767 { 768 struct ccp_device *ccp = (struct ccp_device *)data; 769 770 ccp5_disable_queue_interrupts(ccp); 771 ccp->total_interrupts++; 772 if (ccp->use_tasklet) 773 tasklet_schedule(&ccp->irq_tasklet); 774 else 775 ccp5_irq_bh((unsigned long)ccp); 776 return IRQ_HANDLED; 777 } 778 ccp5_init(struct ccp_device * ccp)779 static int ccp5_init(struct ccp_device *ccp) 780 { 781 struct device *dev = ccp->dev; 782 struct ccp_cmd_queue *cmd_q; 783 struct dma_pool *dma_pool; 784 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 785 unsigned int qmr, i; 786 u64 status; 787 u32 status_lo, status_hi; 788 int ret; 789 790 /* Find available queues */ 791 qmr = ioread32(ccp->io_regs + Q_MASK_REG); 792 /* 793 * Check for a access to the registers. If this read returns 794 * 0xffffffff, it's likely that the system is running a broken 795 * BIOS which disallows access to the device. Stop here and fail 796 * the initialization (but not the load, as the PSP could get 797 * properly initialized). 798 */ 799 if (qmr == 0xffffffff) { 800 dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n"); 801 return 1; 802 } 803 804 for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { 805 if (!(qmr & (1 << i))) 806 continue; 807 808 /* Allocate a dma pool for this queue */ 809 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", 810 ccp->name, i); 811 dma_pool = dma_pool_create(dma_pool_name, dev, 812 CCP_DMAPOOL_MAX_SIZE, 813 CCP_DMAPOOL_ALIGN, 0); 814 if (!dma_pool) { 815 dev_err(dev, "unable to allocate dma pool\n"); 816 ret = -ENOMEM; 817 goto e_pool; 818 } 819 820 cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; 821 ccp->cmd_q_count++; 822 823 cmd_q->ccp = ccp; 824 cmd_q->id = i; 825 cmd_q->dma_pool = dma_pool; 826 mutex_init(&cmd_q->q_mutex); 827 828 /* Page alignment satisfies our needs for N <= 128 */ 829 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); 830 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); 831 cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, 832 &cmd_q->qbase_dma, 833 GFP_KERNEL); 834 if (!cmd_q->qbase) { 835 dev_err(dev, "unable to allocate command queue\n"); 836 ret = -ENOMEM; 837 goto e_pool; 838 } 839 840 cmd_q->qidx = 0; 841 /* Preset some register values and masks that are queue 842 * number dependent 843 */ 844 cmd_q->reg_control = ccp->io_regs + 845 CMD5_Q_STATUS_INCR * (i + 1); 846 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; 847 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; 848 cmd_q->reg_int_enable = cmd_q->reg_control + 849 CMD5_Q_INT_ENABLE_BASE; 850 cmd_q->reg_interrupt_status = cmd_q->reg_control + 851 CMD5_Q_INTERRUPT_STATUS_BASE; 852 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; 853 cmd_q->reg_int_status = cmd_q->reg_control + 854 CMD5_Q_INT_STATUS_BASE; 855 cmd_q->reg_dma_status = cmd_q->reg_control + 856 CMD5_Q_DMA_STATUS_BASE; 857 cmd_q->reg_dma_read_status = cmd_q->reg_control + 858 CMD5_Q_DMA_READ_STATUS_BASE; 859 cmd_q->reg_dma_write_status = cmd_q->reg_control + 860 CMD5_Q_DMA_WRITE_STATUS_BASE; 861 862 init_waitqueue_head(&cmd_q->int_queue); 863 864 dev_dbg(dev, "queue #%u available\n", i); 865 } 866 867 if (ccp->cmd_q_count == 0) { 868 dev_notice(dev, "no command queues available\n"); 869 ret = 1; 870 goto e_pool; 871 } 872 873 /* Turn off the queues and disable interrupts until ready */ 874 ccp5_disable_queue_interrupts(ccp); 875 for (i = 0; i < ccp->cmd_q_count; i++) { 876 cmd_q = &ccp->cmd_q[i]; 877 878 cmd_q->qcontrol = 0; /* Start with nothing */ 879 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 880 881 ioread32(cmd_q->reg_int_status); 882 ioread32(cmd_q->reg_status); 883 884 /* Clear the interrupt status */ 885 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 886 } 887 888 dev_dbg(dev, "Requesting an IRQ...\n"); 889 /* Request an irq */ 890 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp); 891 if (ret) { 892 dev_err(dev, "unable to allocate an IRQ\n"); 893 goto e_pool; 894 } 895 /* Initialize the ISR tasklet */ 896 if (ccp->use_tasklet) 897 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh, 898 (unsigned long)ccp); 899 900 dev_dbg(dev, "Loading LSB map...\n"); 901 /* Copy the private LSB mask to the public registers */ 902 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 903 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 904 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); 905 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); 906 status = ((u64)status_hi<<30) | (u64)status_lo; 907 908 dev_dbg(dev, "Configuring virtual queues...\n"); 909 /* Configure size of each virtual queue accessible to host */ 910 for (i = 0; i < ccp->cmd_q_count; i++) { 911 u32 dma_addr_lo; 912 u32 dma_addr_hi; 913 914 cmd_q = &ccp->cmd_q[i]; 915 916 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); 917 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; 918 919 cmd_q->qdma_tail = cmd_q->qbase_dma; 920 dma_addr_lo = low_address(cmd_q->qdma_tail); 921 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); 922 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); 923 924 dma_addr_hi = high_address(cmd_q->qdma_tail); 925 cmd_q->qcontrol |= (dma_addr_hi << 16); 926 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 927 928 /* Find the LSB regions accessible to the queue */ 929 ccp_find_lsb_regions(cmd_q, status); 930 cmd_q->lsb = -1; /* Unassigned value */ 931 } 932 933 dev_dbg(dev, "Assigning LSBs...\n"); 934 ret = ccp_assign_lsbs(ccp); 935 if (ret) { 936 dev_err(dev, "Unable to assign LSBs (%d)\n", ret); 937 goto e_irq; 938 } 939 940 /* Optimization: pre-allocate LSB slots for each queue */ 941 for (i = 0; i < ccp->cmd_q_count; i++) { 942 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 943 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2); 944 } 945 946 dev_dbg(dev, "Starting threads...\n"); 947 /* Create a kthread for each queue */ 948 for (i = 0; i < ccp->cmd_q_count; i++) { 949 struct task_struct *kthread; 950 951 cmd_q = &ccp->cmd_q[i]; 952 953 kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, 954 "%s-q%u", ccp->name, cmd_q->id); 955 if (IS_ERR(kthread)) { 956 dev_err(dev, "error creating queue thread (%ld)\n", 957 PTR_ERR(kthread)); 958 ret = PTR_ERR(kthread); 959 goto e_kthread; 960 } 961 962 cmd_q->kthread = kthread; 963 } 964 965 dev_dbg(dev, "Enabling interrupts...\n"); 966 ccp5_enable_queue_interrupts(ccp); 967 968 dev_dbg(dev, "Registering device...\n"); 969 /* Put this on the unit list to make it available */ 970 ccp_add_device(ccp); 971 972 ret = ccp_register_rng(ccp); 973 if (ret) 974 goto e_kthread; 975 976 /* Register the DMA engine support */ 977 ret = ccp_dmaengine_register(ccp); 978 if (ret) 979 goto e_hwrng; 980 981 #ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS 982 /* Set up debugfs entries */ 983 ccp5_debugfs_setup(ccp); 984 #endif 985 986 return 0; 987 988 e_hwrng: 989 ccp_unregister_rng(ccp); 990 991 e_kthread: 992 for (i = 0; i < ccp->cmd_q_count; i++) 993 if (ccp->cmd_q[i].kthread) 994 kthread_stop(ccp->cmd_q[i].kthread); 995 996 e_irq: 997 sp_free_ccp_irq(ccp->sp, ccp); 998 999 e_pool: 1000 for (i = 0; i < ccp->cmd_q_count; i++) 1001 dma_pool_destroy(ccp->cmd_q[i].dma_pool); 1002 1003 return ret; 1004 } 1005 ccp5_destroy(struct ccp_device * ccp)1006 static void ccp5_destroy(struct ccp_device *ccp) 1007 { 1008 struct ccp_cmd_queue *cmd_q; 1009 struct ccp_cmd *cmd; 1010 unsigned int i; 1011 1012 /* Unregister the DMA engine */ 1013 ccp_dmaengine_unregister(ccp); 1014 1015 /* Unregister the RNG */ 1016 ccp_unregister_rng(ccp); 1017 1018 /* Remove this device from the list of available units first */ 1019 ccp_del_device(ccp); 1020 1021 #ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS 1022 /* We're in the process of tearing down the entire driver; 1023 * when all the devices are gone clean up debugfs 1024 */ 1025 if (ccp_present()) 1026 ccp5_debugfs_destroy(); 1027 #endif 1028 1029 /* Disable and clear interrupts */ 1030 ccp5_disable_queue_interrupts(ccp); 1031 for (i = 0; i < ccp->cmd_q_count; i++) { 1032 cmd_q = &ccp->cmd_q[i]; 1033 1034 /* Turn off the run bit */ 1035 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); 1036 1037 /* Clear the interrupt status */ 1038 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); 1039 ioread32(cmd_q->reg_int_status); 1040 ioread32(cmd_q->reg_status); 1041 } 1042 1043 /* Stop the queue kthreads */ 1044 for (i = 0; i < ccp->cmd_q_count; i++) 1045 if (ccp->cmd_q[i].kthread) 1046 kthread_stop(ccp->cmd_q[i].kthread); 1047 1048 sp_free_ccp_irq(ccp->sp, ccp); 1049 1050 /* Flush the cmd and backlog queue */ 1051 while (!list_empty(&ccp->cmd)) { 1052 /* Invoke the callback directly with an error code */ 1053 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); 1054 list_del(&cmd->entry); 1055 cmd->callback(cmd->data, -ENODEV); 1056 } 1057 while (!list_empty(&ccp->backlog)) { 1058 /* Invoke the callback directly with an error code */ 1059 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); 1060 list_del(&cmd->entry); 1061 cmd->callback(cmd->data, -ENODEV); 1062 } 1063 } 1064 ccp5_config(struct ccp_device * ccp)1065 static void ccp5_config(struct ccp_device *ccp) 1066 { 1067 /* Public side */ 1068 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); 1069 } 1070 ccp5other_config(struct ccp_device * ccp)1071 static void ccp5other_config(struct ccp_device *ccp) 1072 { 1073 int i; 1074 u32 rnd; 1075 1076 /* We own all of the queues on the NTB CCP */ 1077 1078 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); 1079 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); 1080 for (i = 0; i < 12; i++) { 1081 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); 1082 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); 1083 } 1084 1085 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); 1086 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); 1087 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); 1088 1089 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); 1090 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); 1091 1092 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); 1093 1094 ccp5_config(ccp); 1095 } 1096 1097 /* Version 5 adds some function, but is essentially the same as v5 */ 1098 static const struct ccp_actions ccp5_actions = { 1099 .aes = ccp5_perform_aes, 1100 .xts_aes = ccp5_perform_xts_aes, 1101 .sha = ccp5_perform_sha, 1102 .des3 = ccp5_perform_des3, 1103 .rsa = ccp5_perform_rsa, 1104 .passthru = ccp5_perform_passthru, 1105 .ecc = ccp5_perform_ecc, 1106 .sballoc = ccp_lsb_alloc, 1107 .sbfree = ccp_lsb_free, 1108 .init = ccp5_init, 1109 .destroy = ccp5_destroy, 1110 .get_free_slots = ccp5_get_free_slots, 1111 }; 1112 1113 const struct ccp_vdata ccpv5a = { 1114 .version = CCP_VERSION(5, 0), 1115 .setup = ccp5_config, 1116 .perform = &ccp5_actions, 1117 .offset = 0x0, 1118 .rsamax = CCP5_RSA_MAX_WIDTH, 1119 }; 1120 1121 const struct ccp_vdata ccpv5b = { 1122 .version = CCP_VERSION(5, 0), 1123 .dma_chan_attr = DMA_PRIVATE, 1124 .setup = ccp5other_config, 1125 .perform = &ccp5_actions, 1126 .offset = 0x0, 1127 .rsamax = CCP5_RSA_MAX_WIDTH, 1128 }; 1129
