Lines Matching refs:rctx
135 struct iproc_reqctx_s *rctx, in spu_skcipher_rx_sg_create() argument
141 struct iproc_ctx_s *ctx = rctx->ctx; in spu_skcipher_rx_sg_create()
145 rctx->gfp); in spu_skcipher_rx_sg_create()
152 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_skcipher_rx_sg_create()
157 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, in spu_skcipher_rx_sg_create()
161 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, in spu_skcipher_rx_sg_create()
162 rctx->dst_nents, chunksize); in spu_skcipher_rx_sg_create()
170 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_skcipher_rx_sg_create()
172 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_skcipher_rx_sg_create()
173 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_skcipher_rx_sg_create()
199 struct iproc_reqctx_s *rctx, in spu_skcipher_tx_sg_create() argument
204 struct iproc_ctx_s *ctx = rctx->ctx; in spu_skcipher_tx_sg_create()
209 rctx->gfp); in spu_skcipher_tx_sg_create()
216 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_skcipher_tx_sg_create()
222 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE); in spu_skcipher_tx_sg_create()
225 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_skcipher_tx_sg_create()
226 rctx->src_nents, chunksize); in spu_skcipher_tx_sg_create()
234 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_skcipher_tx_sg_create()
238 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_skcipher_tx_sg_create()
239 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_skcipher_tx_sg_create()
300 static int handle_skcipher_req(struct iproc_reqctx_s *rctx) in handle_skcipher_req() argument
303 struct crypto_async_request *areq = rctx->parent; in handle_skcipher_req()
306 struct iproc_ctx_s *ctx = rctx->ctx; in handle_skcipher_req()
331 cipher_parms.iv_len = rctx->iv_ctr_len; in handle_skcipher_req()
333 mssg = &rctx->mb_mssg; in handle_skcipher_req()
334 chunk_start = rctx->src_sent; in handle_skcipher_req()
335 remaining = rctx->total_todo - chunk_start; in handle_skcipher_req()
344 rctx->src_sent += chunksize; in handle_skcipher_req()
345 rctx->total_sent = rctx->src_sent; in handle_skcipher_req()
348 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); in handle_skcipher_req()
349 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); in handle_skcipher_req()
352 rctx->is_encrypt && chunk_start) in handle_skcipher_req()
357 sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr, in handle_skcipher_req()
358 rctx->iv_ctr_len, in handle_skcipher_req()
359 chunk_start - rctx->iv_ctr_len); in handle_skcipher_req()
361 if (rctx->iv_ctr_len) { in handle_skcipher_req()
363 __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr, in handle_skcipher_req()
364 rctx->iv_ctr_len); in handle_skcipher_req()
368 !rctx->is_encrypt) { in handle_skcipher_req()
373 sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr, in handle_skcipher_req()
374 rctx->iv_ctr_len, in handle_skcipher_req()
375 rctx->src_sent - rctx->iv_ctr_len); in handle_skcipher_req()
386 add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4); in handle_skcipher_req()
396 rctx->src_sent, chunk_start, remaining, chunksize); in handle_skcipher_req()
399 memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, in handle_skcipher_req()
400 sizeof(rctx->msg_buf.bcm_spu_req_hdr)); in handle_skcipher_req()
402 spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_skcipher_req()
403 ctx->spu_req_hdr_len, !(rctx->is_encrypt), in handle_skcipher_req()
414 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0, in handle_skcipher_req()
416 rctx->total_sent, stat_pad_len); in handle_skcipher_req()
419 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_skcipher_req()
422 dump_sg(rctx->src_sg, rctx->src_skip, chunksize); in handle_skcipher_req()
423 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_skcipher_req()
431 mssg->ctx = rctx; /* Will be returned in response */ in handle_skcipher_req()
434 rx_frag_num += rctx->dst_nents; in handle_skcipher_req()
440 err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize, in handle_skcipher_req()
446 tx_frag_num += rctx->src_nents; in handle_skcipher_req()
454 err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize, in handle_skcipher_req()
459 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_skcipher_req()
471 static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) in handle_skcipher_resp() argument
474 struct crypto_async_request *areq = rctx->parent; in handle_skcipher_resp()
476 struct iproc_ctx_s *ctx = rctx->ctx; in handle_skcipher_resp()
480 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); in handle_skcipher_resp()
494 __func__, rctx->total_received, payload_len); in handle_skcipher_resp()
496 dump_sg(req->dst, rctx->total_received, payload_len); in handle_skcipher_resp()
498 rctx->total_received += payload_len; in handle_skcipher_resp()
499 if (rctx->total_received == rctx->total_todo) { in handle_skcipher_resp()
527 struct iproc_reqctx_s *rctx, in spu_ahash_rx_sg_create() argument
533 struct iproc_ctx_s *ctx = rctx->ctx; in spu_ahash_rx_sg_create()
536 rctx->gfp); in spu_ahash_rx_sg_create()
543 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_ahash_rx_sg_create()
546 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); in spu_ahash_rx_sg_create()
549 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_ahash_rx_sg_create()
551 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_ahash_rx_sg_create()
552 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_ahash_rx_sg_create()
579 struct iproc_reqctx_s *rctx, in spu_ahash_tx_sg_create() argument
591 rctx->gfp); in spu_ahash_tx_sg_create()
598 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_ahash_tx_sg_create()
602 sg_set_buf(sg++, rctx->hash_carry, hash_carry_len); in spu_ahash_tx_sg_create()
606 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_ahash_tx_sg_create()
607 rctx->src_nents, new_data_len); in spu_ahash_tx_sg_create()
616 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_ahash_tx_sg_create()
620 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_ahash_tx_sg_create()
621 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_ahash_tx_sg_create()
653 static int handle_ahash_req(struct iproc_reqctx_s *rctx) in handle_ahash_req() argument
656 struct crypto_async_request *areq = rctx->parent; in handle_ahash_req()
661 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ahash_req()
696 rctx->total_todo, rctx->total_sent); in handle_ahash_req()
719 mssg = &rctx->mb_mssg; in handle_ahash_req()
720 chunk_start = rctx->src_sent; in handle_ahash_req()
726 nbytes_to_hash = rctx->total_todo - rctx->total_sent; in handle_ahash_req()
737 if (!rctx->is_final) { in handle_ahash_req()
738 u8 *dest = rctx->hash_carry + rctx->hash_carry_len; in handle_ahash_req()
747 new_len = rem - rctx->hash_carry_len; in handle_ahash_req()
749 rctx->src_sent); in handle_ahash_req()
750 rctx->hash_carry_len = rem; in handle_ahash_req()
752 rctx->hash_carry_len); in handle_ahash_req()
754 rctx->hash_carry, in handle_ahash_req()
755 rctx->hash_carry_len); in handle_ahash_req()
762 local_nbuf = rctx->hash_carry_len; in handle_ahash_req()
763 rctx->hash_carry_len = 0; in handle_ahash_req()
769 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, in handle_ahash_req()
776 hash_parms.type = spu->spu_hash_type(rctx->total_sent); in handle_ahash_req()
783 rctx->total_sent += chunksize; in handle_ahash_req()
785 rctx->src_sent += new_data_len; in handle_ahash_req()
787 if ((rctx->total_sent == rctx->total_todo) && rctx->is_final) in handle_ahash_req()
799 hash_parms.key_buf = rctx->incr_hash; in handle_ahash_req()
806 __func__, rctx->is_final, local_nbuf); in handle_ahash_req()
816 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); in handle_ahash_req()
819 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + in handle_ahash_req()
844 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len, in handle_ahash_req()
846 ctx->auth.mode, rctx->total_sent, in handle_ahash_req()
850 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_ahash_req()
852 packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf); in handle_ahash_req()
854 dump_sg(rctx->src_sg, rctx->src_skip, new_data_len); in handle_ahash_req()
855 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_ahash_req()
863 mssg->ctx = rctx; /* Will be returned in response */ in handle_ahash_req()
866 err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize, in handle_ahash_req()
872 tx_frag_num += rctx->src_nents; in handle_ahash_req()
875 err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, in handle_ahash_req()
880 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_ahash_req()
944 static int ahash_req_done(struct iproc_reqctx_s *rctx) in ahash_req_done() argument
947 struct crypto_async_request *areq = rctx->parent; in ahash_req_done()
949 struct iproc_ctx_s *ctx = rctx->ctx; in ahash_req_done()
952 memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize); in ahash_req_done()
970 if (rctx->is_sw_hmac) { in ahash_req_done()
977 if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) { in ahash_req_done()
994 static void handle_ahash_resp(struct iproc_reqctx_s *rctx) in handle_ahash_resp() argument
996 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ahash_resp()
997 struct crypto_async_request *areq = rctx->parent; in handle_ahash_resp()
1006 memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE); in handle_ahash_resp()
1013 if (rctx->is_final && (rctx->total_sent == rctx->total_todo)) in handle_ahash_resp()
1014 ahash_req_done(rctx); in handle_ahash_resp()
1044 struct iproc_reqctx_s *rctx, in spu_aead_rx_sg_create() argument
1052 struct iproc_ctx_s *ctx = rctx->ctx; in spu_aead_rx_sg_create()
1067 rctx->is_encrypt); in spu_aead_rx_sg_create()
1081 rctx->gfp); in spu_aead_rx_sg_create()
1089 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_aead_rx_sg_create()
1096 memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len); in spu_aead_rx_sg_create()
1097 sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len); in spu_aead_rx_sg_create()
1105 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, in spu_aead_rx_sg_create()
1106 rctx->dst_nents, resp_len); in spu_aead_rx_sg_create()
1116 memset(rctx->msg_buf.a.gcmpad, 0, data_padlen); in spu_aead_rx_sg_create()
1117 sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen); in spu_aead_rx_sg_create()
1121 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); in spu_aead_rx_sg_create()
1125 memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len); in spu_aead_rx_sg_create()
1126 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_aead_rx_sg_create()
1129 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_aead_rx_sg_create()
1130 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_aead_rx_sg_create()
1163 struct iproc_reqctx_s *rctx, in spu_aead_tx_sg_create() argument
1176 struct iproc_ctx_s *ctx = rctx->ctx; in spu_aead_tx_sg_create()
1183 rctx->gfp); in spu_aead_tx_sg_create()
1190 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_aead_tx_sg_create()
1205 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len); in spu_aead_tx_sg_create()
1208 memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len); in spu_aead_tx_sg_create()
1209 sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len); in spu_aead_tx_sg_create()
1217 written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_aead_tx_sg_create()
1218 rctx->src_nents, datalen); in spu_aead_tx_sg_create()
1227 memset(rctx->msg_buf.spu_req_pad, 0, pad_len); in spu_aead_tx_sg_create()
1228 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_aead_tx_sg_create()
1232 sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize); in spu_aead_tx_sg_create()
1236 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_aead_tx_sg_create()
1237 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_aead_tx_sg_create()
1259 static int handle_aead_req(struct iproc_reqctx_s *rctx) in handle_aead_req() argument
1262 struct crypto_async_request *areq = rctx->parent; in handle_aead_req()
1265 struct iproc_ctx_s *ctx = rctx->ctx; in handle_aead_req()
1288 chunksize = rctx->total_todo; in handle_aead_req()
1296 req_opts.is_inbound = !(rctx->is_encrypt); in handle_aead_req()
1306 cipher_parms.iv_buf = rctx->msg_buf.iv_ctr; in handle_aead_req()
1307 cipher_parms.iv_len = rctx->iv_ctr_len; in handle_aead_req()
1329 if (rctx->is_encrypt) { in handle_aead_req()
1344 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); in handle_aead_req()
1345 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); in handle_aead_req()
1347 assoc_nents = spu_sg_count(rctx->assoc, 0, in handle_aead_req()
1350 mssg = &rctx->mb_mssg; in handle_aead_req()
1352 rctx->total_sent = chunksize; in handle_aead_req()
1353 rctx->src_sent = chunksize; in handle_aead_req()
1357 rctx->is_encrypt)) in handle_aead_req()
1361 rctx->iv_ctr_len); in handle_aead_req()
1387 if (!rctx->is_encrypt) in handle_aead_req()
1394 chunksize, rctx->is_encrypt, in handle_aead_req()
1404 if (!rctx->is_encrypt) in handle_aead_req()
1417 if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) { in handle_aead_req()
1421 sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize, in handle_aead_req()
1422 req->assoclen + rctx->total_sent - in handle_aead_req()
1431 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); in handle_aead_req()
1433 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + in handle_aead_req()
1450 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, in handle_aead_req()
1453 rctx->total_sent, stat_pad_len); in handle_aead_req()
1456 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_aead_req()
1458 dump_sg(rctx->assoc, 0, aead_parms.assoc_size); in handle_aead_req()
1459 packet_dump(" aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len); in handle_aead_req()
1461 dump_sg(rctx->src_sg, rctx->src_skip, chunksize); in handle_aead_req()
1462 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_aead_req()
1470 mssg->ctx = rctx; /* Will be returned in response */ in handle_aead_req()
1473 rx_frag_num += rctx->dst_nents; in handle_aead_req()
1484 (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) { in handle_aead_req()
1492 rx_frag_num -= rctx->dst_nents; in handle_aead_req()
1495 err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num, in handle_aead_req()
1503 tx_frag_num += rctx->src_nents; in handle_aead_req()
1511 err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, in handle_aead_req()
1512 rctx->assoc, aead_parms.assoc_size, in handle_aead_req()
1518 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_aead_req()
1529 static void handle_aead_resp(struct iproc_reqctx_s *rctx) in handle_aead_resp() argument
1532 struct crypto_async_request *areq = rctx->parent; in handle_aead_resp()
1535 struct iproc_ctx_s *ctx = rctx->ctx; in handle_aead_resp()
1541 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); in handle_aead_resp()
1548 packet_dump(" assoc_data ", rctx->msg_buf.a.resp_aad, in handle_aead_resp()
1557 if (rctx->is_encrypt) { in handle_aead_resp()
1558 icv_offset = req->assoclen + rctx->total_sent; in handle_aead_resp()
1559 packet_dump(" ICV: ", rctx->msg_buf.digest, ctx->digestsize); in handle_aead_resp()
1561 sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest, in handle_aead_resp()
1589 static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx) in spu_chunk_cleanup() argument
1592 struct brcm_message *mssg = &rctx->mb_mssg; in spu_chunk_cleanup()
1607 static void finish_req(struct iproc_reqctx_s *rctx, int err) in finish_req() argument
1609 struct crypto_async_request *areq = rctx->parent; in finish_req()
1614 spu_chunk_cleanup(rctx); in finish_req()
1629 struct iproc_reqctx_s *rctx; in spu_rx_callback() local
1632 rctx = mssg->ctx; in spu_rx_callback()
1633 if (unlikely(!rctx)) { in spu_rx_callback()
1641 err = spu->spu_status_process(rctx->msg_buf.rx_stat); in spu_rx_callback()
1650 switch (rctx->ctx->alg->type) { in spu_rx_callback()
1652 handle_skcipher_resp(rctx); in spu_rx_callback()
1655 handle_ahash_resp(rctx); in spu_rx_callback()
1658 handle_aead_resp(rctx); in spu_rx_callback()
1669 if (rctx->total_sent < rctx->total_todo) { in spu_rx_callback()
1671 spu_chunk_cleanup(rctx); in spu_rx_callback()
1673 switch (rctx->ctx->alg->type) { in spu_rx_callback()
1675 err = handle_skcipher_req(rctx); in spu_rx_callback()
1678 err = handle_ahash_req(rctx); in spu_rx_callback()
1687 err = handle_aead_req(rctx); in spu_rx_callback()
1699 finish_req(rctx, err); in spu_rx_callback()
1715 struct iproc_reqctx_s *rctx = skcipher_request_ctx(req); in skcipher_enqueue() local
1722 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in skcipher_enqueue()
1724 rctx->parent = &req->base; in skcipher_enqueue()
1725 rctx->is_encrypt = encrypt; in skcipher_enqueue()
1726 rctx->bd_suppress = false; in skcipher_enqueue()
1727 rctx->total_todo = req->cryptlen; in skcipher_enqueue()
1728 rctx->src_sent = 0; in skcipher_enqueue()
1729 rctx->total_sent = 0; in skcipher_enqueue()
1730 rctx->total_received = 0; in skcipher_enqueue()
1731 rctx->ctx = ctx; in skcipher_enqueue()
1734 rctx->src_sg = req->src; in skcipher_enqueue()
1735 rctx->src_nents = 0; in skcipher_enqueue()
1736 rctx->src_skip = 0; in skcipher_enqueue()
1737 rctx->dst_sg = req->dst; in skcipher_enqueue()
1738 rctx->dst_nents = 0; in skcipher_enqueue()
1739 rctx->dst_skip = 0; in skcipher_enqueue()
1747 rctx->iv_ctr_len = in skcipher_enqueue()
1749 memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len); in skcipher_enqueue()
1751 rctx->iv_ctr_len = 0; in skcipher_enqueue()
1755 rctx->chan_idx = select_channel(); in skcipher_enqueue()
1756 err = handle_skcipher_req(rctx); in skcipher_enqueue()
1759 spu_chunk_cleanup(rctx); in skcipher_enqueue()
1905 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_enqueue() local
1913 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in ahash_enqueue()
1915 rctx->parent = &req->base; in ahash_enqueue()
1916 rctx->ctx = ctx; in ahash_enqueue()
1917 rctx->bd_suppress = true; in ahash_enqueue()
1918 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); in ahash_enqueue()
1921 rctx->src_sg = req->src; in ahash_enqueue()
1922 rctx->src_skip = 0; in ahash_enqueue()
1923 rctx->src_nents = 0; in ahash_enqueue()
1924 rctx->dst_sg = NULL; in ahash_enqueue()
1925 rctx->dst_skip = 0; in ahash_enqueue()
1926 rctx->dst_nents = 0; in ahash_enqueue()
1929 if ((rctx->is_final == 1) && (rctx->total_todo == 0) && in ahash_enqueue()
1933 rctx->is_final ? "" : "non-", alg_name); in ahash_enqueue()
1942 rctx->chan_idx = select_channel(); in ahash_enqueue()
1944 err = handle_ahash_req(rctx); in ahash_enqueue()
1947 spu_chunk_cleanup(rctx); in ahash_enqueue()
1962 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_init() local
1969 rctx->hash_carry_len = 0; in __ahash_init()
1970 rctx->is_final = 0; in __ahash_init()
1972 rctx->total_todo = 0; in __ahash_init()
1973 rctx->src_sent = 0; in __ahash_init()
1974 rctx->total_sent = 0; in __ahash_init()
1975 rctx->total_received = 0; in __ahash_init()
1981 rctx->is_sw_hmac = false; in __ahash_init()
2078 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_update() local
2084 rctx->total_todo += req->nbytes; in __ahash_update()
2085 rctx->src_sent = 0; in __ahash_update()
2136 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_final() local
2140 rctx->is_final = 1; in __ahash_final()
2173 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_finup() local
2177 rctx->total_todo += req->nbytes; in __ahash_finup()
2178 rctx->src_sent = 0; in __ahash_finup()
2179 rctx->is_final = 1; in __ahash_finup()
2288 const struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_export() local
2291 spu_exp->total_todo = rctx->total_todo; in ahash_export()
2292 spu_exp->total_sent = rctx->total_sent; in ahash_export()
2293 spu_exp->is_sw_hmac = rctx->is_sw_hmac; in ahash_export()
2294 memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry)); in ahash_export()
2295 spu_exp->hash_carry_len = rctx->hash_carry_len; in ahash_export()
2296 memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash)); in ahash_export()
2303 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_import() local
2306 rctx->total_todo = spu_exp->total_todo; in ahash_import()
2307 rctx->total_sent = spu_exp->total_sent; in ahash_import()
2308 rctx->is_sw_hmac = spu_exp->is_sw_hmac; in ahash_import()
2309 memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry)); in ahash_import()
2310 rctx->hash_carry_len = spu_exp->hash_carry_len; in ahash_import()
2311 memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash)); in ahash_import()
2418 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_hmac_init() local
2431 rctx->is_sw_hmac = true; in ahash_hmac_init()
2434 memcpy(rctx->hash_carry, ctx->ipad, blocksize); in ahash_hmac_init()
2435 rctx->hash_carry_len = blocksize; in ahash_hmac_init()
2436 rctx->total_todo += blocksize; in ahash_hmac_init()
2468 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_hmac_digest() local
2488 rctx->is_sw_hmac = false; in ahash_hmac_digest()
2491 rctx->is_sw_hmac = true; in ahash_hmac_digest()
2494 memcpy(rctx->hash_carry, ctx->ipad, blocksize); in ahash_hmac_digest()
2495 rctx->hash_carry_len = blocksize; in ahash_hmac_digest()
2496 rctx->total_todo += blocksize; in ahash_hmac_digest()
2506 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_need_fallback() local
2519 if ((rctx->is_encrypt && (req->cryptlen == 0)) || in aead_need_fallback()
2520 (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) { in aead_need_fallback()
2554 rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE && in aead_need_fallback()
2577 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_complete() local
2584 areq->complete = rctx->old_complete; in aead_complete()
2585 areq->data = rctx->old_data; in aead_complete()
2594 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_do_fallback() local
2603 rctx->old_tfm = tfm; in aead_do_fallback()
2609 rctx->old_complete = req->base.complete; in aead_do_fallback()
2610 rctx->old_data = req->base.data; in aead_do_fallback()
2622 rctx->old_complete, req); in aead_do_fallback()
2623 req->base.data = rctx->old_data; in aead_do_fallback()
2637 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_enqueue() local
2651 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in aead_enqueue()
2653 rctx->parent = &req->base; in aead_enqueue()
2654 rctx->is_encrypt = is_encrypt; in aead_enqueue()
2655 rctx->bd_suppress = false; in aead_enqueue()
2656 rctx->total_todo = req->cryptlen; in aead_enqueue()
2657 rctx->src_sent = 0; in aead_enqueue()
2658 rctx->total_sent = 0; in aead_enqueue()
2659 rctx->total_received = 0; in aead_enqueue()
2660 rctx->is_sw_hmac = false; in aead_enqueue()
2661 rctx->ctx = ctx; in aead_enqueue()
2662 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); in aead_enqueue()
2665 rctx->assoc = req->src; in aead_enqueue()
2672 if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg, in aead_enqueue()
2673 &rctx->src_skip) < 0) { in aead_enqueue()
2679 rctx->src_nents = 0; in aead_enqueue()
2680 rctx->dst_nents = 0; in aead_enqueue()
2682 rctx->dst_sg = rctx->src_sg; in aead_enqueue()
2683 rctx->dst_skip = rctx->src_skip; in aead_enqueue()
2690 if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg, in aead_enqueue()
2691 &rctx->dst_skip) < 0) { in aead_enqueue()
2703 rctx->iv_ctr_len = in aead_enqueue()
2707 rctx->iv_ctr_len = CCM_AES_IV_SIZE; in aead_enqueue()
2709 rctx->iv_ctr_len = 0; in aead_enqueue()
2712 rctx->hash_carry_len = 0; in aead_enqueue()
2716 rctx->src_sg, rctx->src_skip); in aead_enqueue()
2717 flow_log(" assoc: %p, assoclen %u\n", rctx->assoc, req->assoclen); in aead_enqueue()
2720 rctx->dst_sg, rctx->dst_skip); in aead_enqueue()
2721 flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len); in aead_enqueue()
2722 flow_dump(" iv: ", req->iv, rctx->iv_ctr_len); in aead_enqueue()
2738 if (rctx->iv_ctr_len) { in aead_enqueue()
2740 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset, in aead_enqueue()
2742 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len, in aead_enqueue()
2744 rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset); in aead_enqueue()
2747 rctx->chan_idx = select_channel(); in aead_enqueue()
2748 err = handle_aead_req(rctx); in aead_enqueue()
2751 spu_chunk_cleanup(rctx); in aead_enqueue()