1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux
4 */
5
6 #include <linux/crc32.h>
7 #include <linux/base64.h>
8 #include <linux/prandom.h>
9 #include <asm/unaligned.h>
10 #include <crypto/hash.h>
11 #include <crypto/dh.h>
12 #include "nvme.h"
13 #include "fabrics.h"
14 #include <linux/nvme-auth.h>
15
16 struct nvme_dhchap_queue_context {
17 struct list_head entry;
18 struct work_struct auth_work;
19 struct nvme_ctrl *ctrl;
20 struct crypto_shash *shash_tfm;
21 struct crypto_kpp *dh_tfm;
22 void *buf;
23 size_t buf_size;
24 int qid;
25 int error;
26 u32 s1;
27 u32 s2;
28 u16 transaction;
29 u8 status;
30 u8 hash_id;
31 size_t hash_len;
32 u8 dhgroup_id;
33 u8 c1[64];
34 u8 c2[64];
35 u8 response[64];
36 u8 *host_response;
37 u8 *ctrl_key;
38 int ctrl_key_len;
39 u8 *host_key;
40 int host_key_len;
41 u8 *sess_key;
42 int sess_key_len;
43 };
44
45 #define nvme_auth_flags_from_qid(qid) \
46 (qid == 0) ? 0 : BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED
47 #define nvme_auth_queue_from_qid(ctrl, qid) \
48 (qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q
49
nvme_auth_submit(struct nvme_ctrl * ctrl,int qid,void * data,size_t data_len,bool auth_send)50 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
51 void *data, size_t data_len, bool auth_send)
52 {
53 struct nvme_command cmd = {};
54 blk_mq_req_flags_t flags = nvme_auth_flags_from_qid(qid);
55 struct request_queue *q = nvme_auth_queue_from_qid(ctrl, qid);
56 int ret;
57
58 cmd.auth_common.opcode = nvme_fabrics_command;
59 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER;
60 cmd.auth_common.spsp0 = 0x01;
61 cmd.auth_common.spsp1 = 0x01;
62 if (auth_send) {
63 cmd.auth_send.fctype = nvme_fabrics_type_auth_send;
64 cmd.auth_send.tl = cpu_to_le32(data_len);
65 } else {
66 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive;
67 cmd.auth_receive.al = cpu_to_le32(data_len);
68 }
69
70 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
71 qid == 0 ? NVME_QID_ANY : qid,
72 0, flags);
73 if (ret > 0)
74 dev_warn(ctrl->device,
75 "qid %d auth_send failed with status %d\n", qid, ret);
76 else if (ret < 0)
77 dev_err(ctrl->device,
78 "qid %d auth_send failed with error %d\n", qid, ret);
79 return ret;
80 }
81
nvme_auth_receive_validate(struct nvme_ctrl * ctrl,int qid,struct nvmf_auth_dhchap_failure_data * data,u16 transaction,u8 expected_msg)82 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,
83 struct nvmf_auth_dhchap_failure_data *data,
84 u16 transaction, u8 expected_msg)
85 {
86 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",
87 __func__, qid, data->auth_type, data->auth_id);
88
89 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES &&
90 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
91 return data->rescode_exp;
92 }
93 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES ||
94 data->auth_id != expected_msg) {
95 dev_warn(ctrl->device,
96 "qid %d invalid message %02x/%02x\n",
97 qid, data->auth_type, data->auth_id);
98 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
99 }
100 if (le16_to_cpu(data->t_id) != transaction) {
101 dev_warn(ctrl->device,
102 "qid %d invalid transaction ID %d\n",
103 qid, le16_to_cpu(data->t_id));
104 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
105 }
106 return 0;
107 }
108
nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)109 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,
110 struct nvme_dhchap_queue_context *chap)
111 {
112 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;
113 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);
114
115 if (chap->buf_size < size) {
116 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
117 return -EINVAL;
118 }
119 memset((u8 *)chap->buf, 0, size);
120 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
121 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
122 data->t_id = cpu_to_le16(chap->transaction);
123 data->sc_c = 0; /* No secure channel concatenation */
124 data->napd = 1;
125 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID;
126 data->auth_protocol[0].dhchap.halen = 3;
127 data->auth_protocol[0].dhchap.dhlen = 6;
128 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256;
129 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384;
130 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512;
131 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL;
132 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048;
133 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072;
134 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096;
135 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144;
136 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192;
137
138 return size;
139 }
140
nvme_auth_process_dhchap_challenge(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)141 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,
142 struct nvme_dhchap_queue_context *chap)
143 {
144 struct nvmf_auth_dhchap_challenge_data *data = chap->buf;
145 u16 dhvlen = le16_to_cpu(data->dhvlen);
146 size_t size = sizeof(*data) + data->hl + dhvlen;
147 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);
148 const char *hmac_name, *kpp_name;
149
150 if (chap->buf_size < size) {
151 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
152 return NVME_SC_INVALID_FIELD;
153 }
154
155 hmac_name = nvme_auth_hmac_name(data->hashid);
156 if (!hmac_name) {
157 dev_warn(ctrl->device,
158 "qid %d: invalid HASH ID %d\n",
159 chap->qid, data->hashid);
160 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
161 return NVME_SC_INVALID_FIELD;
162 }
163
164 if (chap->hash_id == data->hashid && chap->shash_tfm &&
165 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) &&
166 crypto_shash_digestsize(chap->shash_tfm) == data->hl) {
167 dev_dbg(ctrl->device,
168 "qid %d: reuse existing hash %s\n",
169 chap->qid, hmac_name);
170 goto select_kpp;
171 }
172
173 /* Reset if hash cannot be reused */
174 if (chap->shash_tfm) {
175 crypto_free_shash(chap->shash_tfm);
176 chap->hash_id = 0;
177 chap->hash_len = 0;
178 }
179 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0,
180 CRYPTO_ALG_ALLOCATES_MEMORY);
181 if (IS_ERR(chap->shash_tfm)) {
182 dev_warn(ctrl->device,
183 "qid %d: failed to allocate hash %s, error %ld\n",
184 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm));
185 chap->shash_tfm = NULL;
186 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
187 return NVME_SC_AUTH_REQUIRED;
188 }
189
190 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) {
191 dev_warn(ctrl->device,
192 "qid %d: invalid hash length %d\n",
193 chap->qid, data->hl);
194 crypto_free_shash(chap->shash_tfm);
195 chap->shash_tfm = NULL;
196 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
197 return NVME_SC_AUTH_REQUIRED;
198 }
199
200 /* Reset host response if the hash had been changed */
201 if (chap->hash_id != data->hashid) {
202 kfree(chap->host_response);
203 chap->host_response = NULL;
204 }
205
206 chap->hash_id = data->hashid;
207 chap->hash_len = data->hl;
208 dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
209 chap->qid, hmac_name);
210
211 select_kpp:
212 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid);
213 if (!kpp_name) {
214 dev_warn(ctrl->device,
215 "qid %d: invalid DH group id %d\n",
216 chap->qid, data->dhgid);
217 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
218 /* Leave previous dh_tfm intact */
219 return NVME_SC_AUTH_REQUIRED;
220 }
221
222 /* Clear host and controller key to avoid accidental reuse */
223 kfree_sensitive(chap->host_key);
224 chap->host_key = NULL;
225 chap->host_key_len = 0;
226 kfree_sensitive(chap->ctrl_key);
227 chap->ctrl_key = NULL;
228 chap->ctrl_key_len = 0;
229
230 if (chap->dhgroup_id == data->dhgid &&
231 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {
232 dev_dbg(ctrl->device,
233 "qid %d: reuse existing DH group %s\n",
234 chap->qid, gid_name);
235 goto skip_kpp;
236 }
237
238 /* Reset dh_tfm if it can't be reused */
239 if (chap->dh_tfm) {
240 crypto_free_kpp(chap->dh_tfm);
241 chap->dh_tfm = NULL;
242 }
243
244 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) {
245 if (dhvlen == 0) {
246 dev_warn(ctrl->device,
247 "qid %d: empty DH value\n",
248 chap->qid);
249 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
250 return NVME_SC_INVALID_FIELD;
251 }
252
253 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0);
254 if (IS_ERR(chap->dh_tfm)) {
255 int ret = PTR_ERR(chap->dh_tfm);
256
257 dev_warn(ctrl->device,
258 "qid %d: error %d initializing DH group %s\n",
259 chap->qid, ret, gid_name);
260 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
261 chap->dh_tfm = NULL;
262 return NVME_SC_AUTH_REQUIRED;
263 }
264 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",
265 chap->qid, gid_name);
266 } else if (dhvlen != 0) {
267 dev_warn(ctrl->device,
268 "qid %d: invalid DH value for NULL DH\n",
269 chap->qid);
270 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
271 return NVME_SC_INVALID_FIELD;
272 }
273 chap->dhgroup_id = data->dhgid;
274
275 skip_kpp:
276 chap->s1 = le32_to_cpu(data->seqnum);
277 memcpy(chap->c1, data->cval, chap->hash_len);
278 if (dhvlen) {
279 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL);
280 if (!chap->ctrl_key) {
281 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
282 return NVME_SC_AUTH_REQUIRED;
283 }
284 chap->ctrl_key_len = dhvlen;
285 memcpy(chap->ctrl_key, data->cval + chap->hash_len,
286 dhvlen);
287 dev_dbg(ctrl->device, "ctrl public key %*ph\n",
288 (int)chap->ctrl_key_len, chap->ctrl_key);
289 }
290
291 return 0;
292 }
293
nvme_auth_set_dhchap_reply_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)294 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,
295 struct nvme_dhchap_queue_context *chap)
296 {
297 struct nvmf_auth_dhchap_reply_data *data = chap->buf;
298 size_t size = sizeof(*data);
299
300 size += 2 * chap->hash_len;
301
302 if (chap->host_key_len)
303 size += chap->host_key_len;
304
305 if (chap->buf_size < size) {
306 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
307 return -EINVAL;
308 }
309
310 memset(chap->buf, 0, size);
311 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
312 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
313 data->t_id = cpu_to_le16(chap->transaction);
314 data->hl = chap->hash_len;
315 data->dhvlen = cpu_to_le16(chap->host_key_len);
316 memcpy(data->rval, chap->response, chap->hash_len);
317 if (ctrl->ctrl_key) {
318 get_random_bytes(chap->c2, chap->hash_len);
319 data->cvalid = 1;
320 chap->s2 = nvme_auth_get_seqnum();
321 memcpy(data->rval + chap->hash_len, chap->c2,
322 chap->hash_len);
323 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",
324 __func__, chap->qid, (int)chap->hash_len, chap->c2);
325 } else {
326 memset(chap->c2, 0, chap->hash_len);
327 chap->s2 = 0;
328 }
329 data->seqnum = cpu_to_le32(chap->s2);
330 if (chap->host_key_len) {
331 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",
332 __func__, chap->qid,
333 chap->host_key_len, chap->host_key);
334 memcpy(data->rval + 2 * chap->hash_len, chap->host_key,
335 chap->host_key_len);
336 }
337
338 return size;
339 }
340
nvme_auth_process_dhchap_success1(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)341 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
342 struct nvme_dhchap_queue_context *chap)
343 {
344 struct nvmf_auth_dhchap_success1_data *data = chap->buf;
345 size_t size = sizeof(*data);
346
347 if (ctrl->ctrl_key)
348 size += chap->hash_len;
349
350 if (chap->buf_size < size) {
351 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
352 return NVME_SC_INVALID_FIELD;
353 }
354
355 if (data->hl != chap->hash_len) {
356 dev_warn(ctrl->device,
357 "qid %d: invalid hash length %u\n",
358 chap->qid, data->hl);
359 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
360 return NVME_SC_INVALID_FIELD;
361 }
362
363 /* Just print out information for the admin queue */
364 if (chap->qid == 0)
365 dev_info(ctrl->device,
366 "qid 0: authenticated with hash %s dhgroup %s\n",
367 nvme_auth_hmac_name(chap->hash_id),
368 nvme_auth_dhgroup_name(chap->dhgroup_id));
369
370 if (!data->rvalid)
371 return 0;
372
373 /* Validate controller response */
374 if (memcmp(chap->response, data->rval, data->hl)) {
375 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",
376 __func__, chap->qid, (int)chap->hash_len, data->rval);
377 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",
378 __func__, chap->qid, (int)chap->hash_len,
379 chap->response);
380 dev_warn(ctrl->device,
381 "qid %d: controller authentication failed\n",
382 chap->qid);
383 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
384 return NVME_SC_AUTH_REQUIRED;
385 }
386
387 /* Just print out information for the admin queue */
388 if (chap->qid == 0)
389 dev_info(ctrl->device,
390 "qid 0: controller authenticated\n");
391 return 0;
392 }
393
nvme_auth_set_dhchap_success2_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)394 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,
395 struct nvme_dhchap_queue_context *chap)
396 {
397 struct nvmf_auth_dhchap_success2_data *data = chap->buf;
398 size_t size = sizeof(*data);
399
400 memset(chap->buf, 0, size);
401 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
402 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
403 data->t_id = cpu_to_le16(chap->transaction);
404
405 return size;
406 }
407
nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)408 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,
409 struct nvme_dhchap_queue_context *chap)
410 {
411 struct nvmf_auth_dhchap_failure_data *data = chap->buf;
412 size_t size = sizeof(*data);
413
414 memset(chap->buf, 0, size);
415 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
416 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
417 data->t_id = cpu_to_le16(chap->transaction);
418 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
419 data->rescode_exp = chap->status;
420
421 return size;
422 }
423
nvme_auth_dhchap_setup_host_response(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)424 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,
425 struct nvme_dhchap_queue_context *chap)
426 {
427 SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
428 u8 buf[4], *challenge = chap->c1;
429 int ret;
430
431 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",
432 __func__, chap->qid, chap->s1, chap->transaction);
433
434 if (!chap->host_response) {
435 chap->host_response = nvme_auth_transform_key(ctrl->host_key,
436 ctrl->opts->host->nqn);
437 if (IS_ERR(chap->host_response)) {
438 ret = PTR_ERR(chap->host_response);
439 chap->host_response = NULL;
440 return ret;
441 }
442 } else {
443 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",
444 __func__, chap->qid);
445 }
446
447 ret = crypto_shash_setkey(chap->shash_tfm,
448 chap->host_response, ctrl->host_key->len);
449 if (ret) {
450 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
451 chap->qid, ret);
452 goto out;
453 }
454
455 if (chap->dh_tfm) {
456 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
457 if (!challenge) {
458 ret = -ENOMEM;
459 goto out;
460 }
461 ret = nvme_auth_augmented_challenge(chap->hash_id,
462 chap->sess_key,
463 chap->sess_key_len,
464 chap->c1, challenge,
465 chap->hash_len);
466 if (ret)
467 goto out;
468 }
469
470 shash->tfm = chap->shash_tfm;
471 ret = crypto_shash_init(shash);
472 if (ret)
473 goto out;
474 ret = crypto_shash_update(shash, challenge, chap->hash_len);
475 if (ret)
476 goto out;
477 put_unaligned_le32(chap->s1, buf);
478 ret = crypto_shash_update(shash, buf, 4);
479 if (ret)
480 goto out;
481 put_unaligned_le16(chap->transaction, buf);
482 ret = crypto_shash_update(shash, buf, 2);
483 if (ret)
484 goto out;
485 memset(buf, 0, sizeof(buf));
486 ret = crypto_shash_update(shash, buf, 1);
487 if (ret)
488 goto out;
489 ret = crypto_shash_update(shash, "HostHost", 8);
490 if (ret)
491 goto out;
492 ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
493 strlen(ctrl->opts->host->nqn));
494 if (ret)
495 goto out;
496 ret = crypto_shash_update(shash, buf, 1);
497 if (ret)
498 goto out;
499 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
500 strlen(ctrl->opts->subsysnqn));
501 if (ret)
502 goto out;
503 ret = crypto_shash_final(shash, chap->response);
504 out:
505 if (challenge != chap->c1)
506 kfree(challenge);
507 return ret;
508 }
509
nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)510 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
511 struct nvme_dhchap_queue_context *chap)
512 {
513 SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
514 u8 *ctrl_response;
515 u8 buf[4], *challenge = chap->c2;
516 int ret;
517
518 ctrl_response = nvme_auth_transform_key(ctrl->ctrl_key,
519 ctrl->opts->subsysnqn);
520 if (IS_ERR(ctrl_response)) {
521 ret = PTR_ERR(ctrl_response);
522 return ret;
523 }
524 ret = crypto_shash_setkey(chap->shash_tfm,
525 ctrl_response, ctrl->ctrl_key->len);
526 if (ret) {
527 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
528 chap->qid, ret);
529 goto out;
530 }
531
532 if (chap->dh_tfm) {
533 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
534 if (!challenge) {
535 ret = -ENOMEM;
536 goto out;
537 }
538 ret = nvme_auth_augmented_challenge(chap->hash_id,
539 chap->sess_key,
540 chap->sess_key_len,
541 chap->c2, challenge,
542 chap->hash_len);
543 if (ret)
544 goto out;
545 }
546 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",
547 __func__, chap->qid, chap->s2, chap->transaction);
548 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",
549 __func__, chap->qid, (int)chap->hash_len, challenge);
550 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",
551 __func__, chap->qid, ctrl->opts->subsysnqn);
552 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",
553 __func__, chap->qid, ctrl->opts->host->nqn);
554 shash->tfm = chap->shash_tfm;
555 ret = crypto_shash_init(shash);
556 if (ret)
557 goto out;
558 ret = crypto_shash_update(shash, challenge, chap->hash_len);
559 if (ret)
560 goto out;
561 put_unaligned_le32(chap->s2, buf);
562 ret = crypto_shash_update(shash, buf, 4);
563 if (ret)
564 goto out;
565 put_unaligned_le16(chap->transaction, buf);
566 ret = crypto_shash_update(shash, buf, 2);
567 if (ret)
568 goto out;
569 memset(buf, 0, 4);
570 ret = crypto_shash_update(shash, buf, 1);
571 if (ret)
572 goto out;
573 ret = crypto_shash_update(shash, "Controller", 10);
574 if (ret)
575 goto out;
576 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
577 strlen(ctrl->opts->subsysnqn));
578 if (ret)
579 goto out;
580 ret = crypto_shash_update(shash, buf, 1);
581 if (ret)
582 goto out;
583 ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
584 strlen(ctrl->opts->host->nqn));
585 if (ret)
586 goto out;
587 ret = crypto_shash_final(shash, chap->response);
588 out:
589 if (challenge != chap->c2)
590 kfree(challenge);
591 kfree(ctrl_response);
592 return ret;
593 }
594
nvme_auth_dhchap_exponential(struct nvme_ctrl * ctrl,struct nvme_dhchap_queue_context * chap)595 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
596 struct nvme_dhchap_queue_context *chap)
597 {
598 int ret;
599
600 if (chap->host_key && chap->host_key_len) {
601 dev_dbg(ctrl->device,
602 "qid %d: reusing host key\n", chap->qid);
603 goto gen_sesskey;
604 }
605 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id);
606 if (ret < 0) {
607 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
608 return ret;
609 }
610
611 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm);
612
613 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL);
614 if (!chap->host_key) {
615 chap->host_key_len = 0;
616 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
617 return -ENOMEM;
618 }
619 ret = nvme_auth_gen_pubkey(chap->dh_tfm,
620 chap->host_key, chap->host_key_len);
621 if (ret) {
622 dev_dbg(ctrl->device,
623 "failed to generate public key, error %d\n", ret);
624 kfree(chap->host_key);
625 chap->host_key = NULL;
626 chap->host_key_len = 0;
627 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
628 return ret;
629 }
630
631 gen_sesskey:
632 chap->sess_key_len = chap->host_key_len;
633 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
634 if (!chap->sess_key) {
635 chap->sess_key_len = 0;
636 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
637 return -ENOMEM;
638 }
639
640 ret = nvme_auth_gen_shared_secret(chap->dh_tfm,
641 chap->ctrl_key, chap->ctrl_key_len,
642 chap->sess_key, chap->sess_key_len);
643 if (ret) {
644 dev_dbg(ctrl->device,
645 "failed to generate shared secret, error %d\n", ret);
646 kfree_sensitive(chap->sess_key);
647 chap->sess_key = NULL;
648 chap->sess_key_len = 0;
649 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
650 return ret;
651 }
652 dev_dbg(ctrl->device, "shared secret %*ph\n",
653 (int)chap->sess_key_len, chap->sess_key);
654 return 0;
655 }
656
__nvme_auth_reset(struct nvme_dhchap_queue_context * chap)657 static void __nvme_auth_reset(struct nvme_dhchap_queue_context *chap)
658 {
659 kfree_sensitive(chap->host_response);
660 chap->host_response = NULL;
661 kfree_sensitive(chap->host_key);
662 chap->host_key = NULL;
663 chap->host_key_len = 0;
664 kfree_sensitive(chap->ctrl_key);
665 chap->ctrl_key = NULL;
666 chap->ctrl_key_len = 0;
667 kfree_sensitive(chap->sess_key);
668 chap->sess_key = NULL;
669 chap->sess_key_len = 0;
670 chap->status = 0;
671 chap->error = 0;
672 chap->s1 = 0;
673 chap->s2 = 0;
674 chap->transaction = 0;
675 memset(chap->c1, 0, sizeof(chap->c1));
676 memset(chap->c2, 0, sizeof(chap->c2));
677 }
678
__nvme_auth_free(struct nvme_dhchap_queue_context * chap)679 static void __nvme_auth_free(struct nvme_dhchap_queue_context *chap)
680 {
681 __nvme_auth_reset(chap);
682 if (chap->shash_tfm)
683 crypto_free_shash(chap->shash_tfm);
684 if (chap->dh_tfm)
685 crypto_free_kpp(chap->dh_tfm);
686 kfree_sensitive(chap->ctrl_key);
687 kfree_sensitive(chap->host_key);
688 kfree_sensitive(chap->sess_key);
689 kfree_sensitive(chap->host_response);
690 kfree(chap->buf);
691 kfree(chap);
692 }
693
__nvme_auth_work(struct work_struct * work)694 static void __nvme_auth_work(struct work_struct *work)
695 {
696 struct nvme_dhchap_queue_context *chap =
697 container_of(work, struct nvme_dhchap_queue_context, auth_work);
698 struct nvme_ctrl *ctrl = chap->ctrl;
699 size_t tl;
700 int ret = 0;
701
702 chap->transaction = ctrl->transaction++;
703
704 /* DH-HMAC-CHAP Step 1: send negotiate */
705 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",
706 __func__, chap->qid);
707 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);
708 if (ret < 0) {
709 chap->error = ret;
710 return;
711 }
712 tl = ret;
713 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
714 if (ret) {
715 chap->error = ret;
716 return;
717 }
718
719 /* DH-HMAC-CHAP Step 2: receive challenge */
720 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",
721 __func__, chap->qid);
722
723 memset(chap->buf, 0, chap->buf_size);
724 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, chap->buf_size, false);
725 if (ret) {
726 dev_warn(ctrl->device,
727 "qid %d failed to receive challenge, %s %d\n",
728 chap->qid, ret < 0 ? "error" : "nvme status", ret);
729 chap->error = ret;
730 return;
731 }
732 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,
733 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE);
734 if (ret) {
735 chap->status = ret;
736 chap->error = NVME_SC_AUTH_REQUIRED;
737 return;
738 }
739
740 ret = nvme_auth_process_dhchap_challenge(ctrl, chap);
741 if (ret) {
742 /* Invalid challenge parameters */
743 chap->error = ret;
744 goto fail2;
745 }
746
747 if (chap->ctrl_key_len) {
748 dev_dbg(ctrl->device,
749 "%s: qid %d DH exponential\n",
750 __func__, chap->qid);
751 ret = nvme_auth_dhchap_exponential(ctrl, chap);
752 if (ret) {
753 chap->error = ret;
754 goto fail2;
755 }
756 }
757
758 dev_dbg(ctrl->device, "%s: qid %d host response\n",
759 __func__, chap->qid);
760 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);
761 if (ret) {
762 chap->error = ret;
763 goto fail2;
764 }
765
766 /* DH-HMAC-CHAP Step 3: send reply */
767 dev_dbg(ctrl->device, "%s: qid %d send reply\n",
768 __func__, chap->qid);
769 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);
770 if (ret < 0) {
771 chap->error = ret;
772 goto fail2;
773 }
774
775 tl = ret;
776 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
777 if (ret) {
778 chap->error = ret;
779 goto fail2;
780 }
781
782 /* DH-HMAC-CHAP Step 4: receive success1 */
783 dev_dbg(ctrl->device, "%s: qid %d receive success1\n",
784 __func__, chap->qid);
785
786 memset(chap->buf, 0, chap->buf_size);
787 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, chap->buf_size, false);
788 if (ret) {
789 dev_warn(ctrl->device,
790 "qid %d failed to receive success1, %s %d\n",
791 chap->qid, ret < 0 ? "error" : "nvme status", ret);
792 chap->error = ret;
793 return;
794 }
795 ret = nvme_auth_receive_validate(ctrl, chap->qid,
796 chap->buf, chap->transaction,
797 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1);
798 if (ret) {
799 chap->status = ret;
800 chap->error = NVME_SC_AUTH_REQUIRED;
801 return;
802 }
803
804 if (ctrl->ctrl_key) {
805 dev_dbg(ctrl->device,
806 "%s: qid %d controller response\n",
807 __func__, chap->qid);
808 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);
809 if (ret) {
810 chap->error = ret;
811 goto fail2;
812 }
813 }
814
815 ret = nvme_auth_process_dhchap_success1(ctrl, chap);
816 if (ret) {
817 /* Controller authentication failed */
818 chap->error = NVME_SC_AUTH_REQUIRED;
819 goto fail2;
820 }
821
822 if (ctrl->ctrl_key) {
823 /* DH-HMAC-CHAP Step 5: send success2 */
824 dev_dbg(ctrl->device, "%s: qid %d send success2\n",
825 __func__, chap->qid);
826 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);
827 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
828 if (ret)
829 chap->error = ret;
830 }
831 if (!ret) {
832 chap->error = 0;
833 return;
834 }
835
836 fail2:
837 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",
838 __func__, chap->qid, chap->status);
839 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);
840 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
841 /*
842 * only update error if send failure2 failed and no other
843 * error had been set during authentication.
844 */
845 if (ret && !chap->error)
846 chap->error = ret;
847 }
848
nvme_auth_negotiate(struct nvme_ctrl * ctrl,int qid)849 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
850 {
851 struct nvme_dhchap_queue_context *chap;
852
853 if (!ctrl->host_key) {
854 dev_warn(ctrl->device, "qid %d: no key\n", qid);
855 return -ENOKEY;
856 }
857
858 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {
859 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);
860 return -ENOKEY;
861 }
862
863 mutex_lock(&ctrl->dhchap_auth_mutex);
864 /* Check if the context is already queued */
865 list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) {
866 WARN_ON(!chap->buf);
867 if (chap->qid == qid) {
868 dev_dbg(ctrl->device, "qid %d: re-using context\n", qid);
869 mutex_unlock(&ctrl->dhchap_auth_mutex);
870 flush_work(&chap->auth_work);
871 __nvme_auth_reset(chap);
872 queue_work(nvme_wq, &chap->auth_work);
873 return 0;
874 }
875 }
876 chap = kzalloc(sizeof(*chap), GFP_KERNEL);
877 if (!chap) {
878 mutex_unlock(&ctrl->dhchap_auth_mutex);
879 return -ENOMEM;
880 }
881 chap->qid = (qid == NVME_QID_ANY) ? 0 : qid;
882 chap->ctrl = ctrl;
883
884 /*
885 * Allocate a large enough buffer for the entire negotiation:
886 * 4k should be enough to ffdhe8192.
887 */
888 chap->buf_size = 4096;
889 chap->buf = kzalloc(chap->buf_size, GFP_KERNEL);
890 if (!chap->buf) {
891 mutex_unlock(&ctrl->dhchap_auth_mutex);
892 kfree(chap);
893 return -ENOMEM;
894 }
895
896 INIT_WORK(&chap->auth_work, __nvme_auth_work);
897 list_add(&chap->entry, &ctrl->dhchap_auth_list);
898 mutex_unlock(&ctrl->dhchap_auth_mutex);
899 queue_work(nvme_wq, &chap->auth_work);
900 return 0;
901 }
902 EXPORT_SYMBOL_GPL(nvme_auth_negotiate);
903
nvme_auth_wait(struct nvme_ctrl * ctrl,int qid)904 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
905 {
906 struct nvme_dhchap_queue_context *chap;
907 int ret;
908
909 mutex_lock(&ctrl->dhchap_auth_mutex);
910 list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) {
911 if (chap->qid != qid)
912 continue;
913 mutex_unlock(&ctrl->dhchap_auth_mutex);
914 flush_work(&chap->auth_work);
915 ret = chap->error;
916 return ret;
917 }
918 mutex_unlock(&ctrl->dhchap_auth_mutex);
919 return -ENXIO;
920 }
921 EXPORT_SYMBOL_GPL(nvme_auth_wait);
922
nvme_auth_reset(struct nvme_ctrl * ctrl)923 void nvme_auth_reset(struct nvme_ctrl *ctrl)
924 {
925 struct nvme_dhchap_queue_context *chap;
926
927 mutex_lock(&ctrl->dhchap_auth_mutex);
928 list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) {
929 mutex_unlock(&ctrl->dhchap_auth_mutex);
930 flush_work(&chap->auth_work);
931 __nvme_auth_reset(chap);
932 }
933 mutex_unlock(&ctrl->dhchap_auth_mutex);
934 }
935 EXPORT_SYMBOL_GPL(nvme_auth_reset);
936
nvme_dhchap_auth_work(struct work_struct * work)937 static void nvme_dhchap_auth_work(struct work_struct *work)
938 {
939 struct nvme_ctrl *ctrl =
940 container_of(work, struct nvme_ctrl, dhchap_auth_work);
941 int ret, q;
942
943 /* Authenticate admin queue first */
944 ret = nvme_auth_negotiate(ctrl, 0);
945 if (ret) {
946 dev_warn(ctrl->device,
947 "qid 0: error %d setting up authentication\n", ret);
948 return;
949 }
950 ret = nvme_auth_wait(ctrl, 0);
951 if (ret) {
952 dev_warn(ctrl->device,
953 "qid 0: authentication failed\n");
954 return;
955 }
956
957 for (q = 1; q < ctrl->queue_count; q++) {
958 ret = nvme_auth_negotiate(ctrl, q);
959 if (ret) {
960 dev_warn(ctrl->device,
961 "qid %d: error %d setting up authentication\n",
962 q, ret);
963 break;
964 }
965 }
966
967 /*
968 * Failure is a soft-state; credentials remain valid until
969 * the controller terminates the connection.
970 */
971 }
972
nvme_auth_init_ctrl(struct nvme_ctrl * ctrl)973 void nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)
974 {
975 INIT_LIST_HEAD(&ctrl->dhchap_auth_list);
976 INIT_WORK(&ctrl->dhchap_auth_work, nvme_dhchap_auth_work);
977 mutex_init(&ctrl->dhchap_auth_mutex);
978 if (!ctrl->opts)
979 return;
980 nvme_auth_generate_key(ctrl->opts->dhchap_secret, &ctrl->host_key);
981 nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, &ctrl->ctrl_key);
982 }
983 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);
984
nvme_auth_stop(struct nvme_ctrl * ctrl)985 void nvme_auth_stop(struct nvme_ctrl *ctrl)
986 {
987 struct nvme_dhchap_queue_context *chap = NULL, *tmp;
988
989 cancel_work_sync(&ctrl->dhchap_auth_work);
990 mutex_lock(&ctrl->dhchap_auth_mutex);
991 list_for_each_entry_safe(chap, tmp, &ctrl->dhchap_auth_list, entry)
992 cancel_work_sync(&chap->auth_work);
993 mutex_unlock(&ctrl->dhchap_auth_mutex);
994 }
995 EXPORT_SYMBOL_GPL(nvme_auth_stop);
996
nvme_auth_free(struct nvme_ctrl * ctrl)997 void nvme_auth_free(struct nvme_ctrl *ctrl)
998 {
999 struct nvme_dhchap_queue_context *chap = NULL, *tmp;
1000
1001 mutex_lock(&ctrl->dhchap_auth_mutex);
1002 list_for_each_entry_safe(chap, tmp, &ctrl->dhchap_auth_list, entry) {
1003 list_del_init(&chap->entry);
1004 flush_work(&chap->auth_work);
1005 __nvme_auth_free(chap);
1006 }
1007 mutex_unlock(&ctrl->dhchap_auth_mutex);
1008 if (ctrl->host_key) {
1009 nvme_auth_free_key(ctrl->host_key);
1010 ctrl->host_key = NULL;
1011 }
1012 if (ctrl->ctrl_key) {
1013 nvme_auth_free_key(ctrl->ctrl_key);
1014 ctrl->ctrl_key = NULL;
1015 }
1016 }
1017 EXPORT_SYMBOL_GPL(nvme_auth_free);
1018