1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVMe over Fabrics DH-HMAC-CHAP authentication command handling.
4  * Copyright (c) 2020 Hannes Reinecke, SUSE Software Solutions.
5  * All rights reserved.
6  */
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/blkdev.h>
9 #include <linux/random.h>
10 #include <linux/nvme-auth.h>
11 #include <crypto/hash.h>
12 #include <crypto/kpp.h>
13 #include "nvmet.h"
14 
nvmet_auth_expired_work(struct work_struct * work)15 static void nvmet_auth_expired_work(struct work_struct *work)
16 {
17 	struct nvmet_sq *sq = container_of(to_delayed_work(work),
18 			struct nvmet_sq, auth_expired_work);
19 
20 	pr_debug("%s: ctrl %d qid %d transaction %u expired, resetting\n",
21 		 __func__, sq->ctrl->cntlid, sq->qid, sq->dhchap_tid);
22 	sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
23 	sq->dhchap_tid = -1;
24 }
25 
nvmet_auth_sq_init(struct nvmet_sq * sq)26 void nvmet_auth_sq_init(struct nvmet_sq *sq)
27 {
28 	/* Initialize in-band authentication */
29 	INIT_DELAYED_WORK(&sq->auth_expired_work, nvmet_auth_expired_work);
30 	sq->authenticated = false;
31 	sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
32 }
33 
nvmet_auth_negotiate(struct nvmet_req * req,void * d)34 static u16 nvmet_auth_negotiate(struct nvmet_req *req, void *d)
35 {
36 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
37 	struct nvmf_auth_dhchap_negotiate_data *data = d;
38 	int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid;
39 
40 	pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n",
41 		 __func__, ctrl->cntlid, req->sq->qid,
42 		 data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid,
43 		 data->auth_protocol[0].dhchap.halen,
44 		 data->auth_protocol[0].dhchap.dhlen);
45 	req->sq->dhchap_tid = le16_to_cpu(data->t_id);
46 	if (data->sc_c)
47 		return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
48 
49 	if (data->napd != 1)
50 		return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
51 
52 	if (data->auth_protocol[0].dhchap.authid !=
53 	    NVME_AUTH_DHCHAP_AUTH_ID)
54 		return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
55 
56 	for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) {
57 		u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i];
58 
59 		if (!fallback_hash_id &&
60 		    crypto_has_shash(nvme_auth_hmac_name(host_hmac_id), 0, 0))
61 			fallback_hash_id = host_hmac_id;
62 		if (ctrl->shash_id != host_hmac_id)
63 			continue;
64 		hash_id = ctrl->shash_id;
65 		break;
66 	}
67 	if (hash_id == 0) {
68 		if (fallback_hash_id == 0) {
69 			pr_debug("%s: ctrl %d qid %d: no usable hash found\n",
70 				 __func__, ctrl->cntlid, req->sq->qid);
71 			return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
72 		}
73 		pr_debug("%s: ctrl %d qid %d: no usable hash found, falling back to %s\n",
74 			 __func__, ctrl->cntlid, req->sq->qid,
75 			 nvme_auth_hmac_name(fallback_hash_id));
76 		ctrl->shash_id = fallback_hash_id;
77 	}
78 
79 	dhgid = -1;
80 	fallback_dhgid = -1;
81 	for (i = 0; i < data->auth_protocol[0].dhchap.dhlen; i++) {
82 		int tmp_dhgid = data->auth_protocol[0].dhchap.idlist[i + 30];
83 
84 		if (tmp_dhgid != ctrl->dh_gid) {
85 			dhgid = tmp_dhgid;
86 			break;
87 		}
88 		if (fallback_dhgid < 0) {
89 			const char *kpp = nvme_auth_dhgroup_kpp(tmp_dhgid);
90 
91 			if (crypto_has_kpp(kpp, 0, 0))
92 				fallback_dhgid = tmp_dhgid;
93 		}
94 	}
95 	if (dhgid < 0) {
96 		if (fallback_dhgid < 0) {
97 			pr_debug("%s: ctrl %d qid %d: no usable DH group found\n",
98 				 __func__, ctrl->cntlid, req->sq->qid);
99 			return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
100 		}
101 		pr_debug("%s: ctrl %d qid %d: configured DH group %s not found\n",
102 			 __func__, ctrl->cntlid, req->sq->qid,
103 			 nvme_auth_dhgroup_name(fallback_dhgid));
104 		ctrl->dh_gid = fallback_dhgid;
105 	}
106 	pr_debug("%s: ctrl %d qid %d: selected DH group %s (%d)\n",
107 		 __func__, ctrl->cntlid, req->sq->qid,
108 		 nvme_auth_dhgroup_name(ctrl->dh_gid), ctrl->dh_gid);
109 	return 0;
110 }
111 
nvmet_auth_reply(struct nvmet_req * req,void * d)112 static u16 nvmet_auth_reply(struct nvmet_req *req, void *d)
113 {
114 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
115 	struct nvmf_auth_dhchap_reply_data *data = d;
116 	u16 dhvlen = le16_to_cpu(data->dhvlen);
117 	u8 *response;
118 
119 	pr_debug("%s: ctrl %d qid %d: data hl %d cvalid %d dhvlen %u\n",
120 		 __func__, ctrl->cntlid, req->sq->qid,
121 		 data->hl, data->cvalid, dhvlen);
122 
123 	if (dhvlen) {
124 		if (!ctrl->dh_tfm)
125 			return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
126 		if (nvmet_auth_ctrl_sesskey(req, data->rval + 2 * data->hl,
127 					    dhvlen) < 0)
128 			return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
129 	}
130 
131 	response = kmalloc(data->hl, GFP_KERNEL);
132 	if (!response)
133 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
134 
135 	if (!ctrl->host_key) {
136 		pr_warn("ctrl %d qid %d no host key\n",
137 			ctrl->cntlid, req->sq->qid);
138 		kfree(response);
139 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
140 	}
141 	if (nvmet_auth_host_hash(req, response, data->hl) < 0) {
142 		pr_debug("ctrl %d qid %d host hash failed\n",
143 			 ctrl->cntlid, req->sq->qid);
144 		kfree(response);
145 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
146 	}
147 
148 	if (memcmp(data->rval, response, data->hl)) {
149 		pr_info("ctrl %d qid %d host response mismatch\n",
150 			ctrl->cntlid, req->sq->qid);
151 		kfree(response);
152 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
153 	}
154 	kfree(response);
155 	pr_debug("%s: ctrl %d qid %d host authenticated\n",
156 		 __func__, ctrl->cntlid, req->sq->qid);
157 	if (data->cvalid) {
158 		req->sq->dhchap_c2 = kmemdup(data->rval + data->hl, data->hl,
159 					     GFP_KERNEL);
160 		if (!req->sq->dhchap_c2)
161 			return NVME_AUTH_DHCHAP_FAILURE_FAILED;
162 
163 		pr_debug("%s: ctrl %d qid %d challenge %*ph\n",
164 			 __func__, ctrl->cntlid, req->sq->qid, data->hl,
165 			 req->sq->dhchap_c2);
166 		req->sq->dhchap_s2 = le32_to_cpu(data->seqnum);
167 	} else {
168 		req->sq->authenticated = true;
169 		req->sq->dhchap_c2 = NULL;
170 	}
171 
172 	return 0;
173 }
174 
nvmet_auth_failure2(void * d)175 static u16 nvmet_auth_failure2(void *d)
176 {
177 	struct nvmf_auth_dhchap_failure_data *data = d;
178 
179 	return data->rescode_exp;
180 }
181 
nvmet_execute_auth_send(struct nvmet_req * req)182 void nvmet_execute_auth_send(struct nvmet_req *req)
183 {
184 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
185 	struct nvmf_auth_dhchap_success2_data *data;
186 	void *d;
187 	u32 tl;
188 	u16 status = 0;
189 
190 	if (req->cmd->auth_send.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
191 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
192 		req->error_loc =
193 			offsetof(struct nvmf_auth_send_command, secp);
194 		goto done;
195 	}
196 	if (req->cmd->auth_send.spsp0 != 0x01) {
197 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
198 		req->error_loc =
199 			offsetof(struct nvmf_auth_send_command, spsp0);
200 		goto done;
201 	}
202 	if (req->cmd->auth_send.spsp1 != 0x01) {
203 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
204 		req->error_loc =
205 			offsetof(struct nvmf_auth_send_command, spsp1);
206 		goto done;
207 	}
208 	tl = le32_to_cpu(req->cmd->auth_send.tl);
209 	if (!tl) {
210 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
211 		req->error_loc =
212 			offsetof(struct nvmf_auth_send_command, tl);
213 		goto done;
214 	}
215 	if (!nvmet_check_transfer_len(req, tl)) {
216 		pr_debug("%s: transfer length mismatch (%u)\n", __func__, tl);
217 		return;
218 	}
219 
220 	d = kmalloc(tl, GFP_KERNEL);
221 	if (!d) {
222 		status = NVME_SC_INTERNAL;
223 		goto done;
224 	}
225 
226 	status = nvmet_copy_from_sgl(req, 0, d, tl);
227 	if (status)
228 		goto done_kfree;
229 
230 	data = d;
231 	pr_debug("%s: ctrl %d qid %d type %d id %d step %x\n", __func__,
232 		 ctrl->cntlid, req->sq->qid, data->auth_type, data->auth_id,
233 		 req->sq->dhchap_step);
234 	if (data->auth_type != NVME_AUTH_COMMON_MESSAGES &&
235 	    data->auth_type != NVME_AUTH_DHCHAP_MESSAGES)
236 		goto done_failure1;
237 	if (data->auth_type == NVME_AUTH_COMMON_MESSAGES) {
238 		if (data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE) {
239 			/* Restart negotiation */
240 			pr_debug("%s: ctrl %d qid %d reset negotiation\n", __func__,
241 				 ctrl->cntlid, req->sq->qid);
242 			if (!req->sq->qid) {
243 				if (nvmet_setup_auth(ctrl) < 0) {
244 					status = NVME_SC_INTERNAL;
245 					pr_err("ctrl %d qid 0 failed to setup"
246 					       "re-authentication",
247 					       ctrl->cntlid);
248 					goto done_failure1;
249 				}
250 			}
251 			req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
252 		} else if (data->auth_id != req->sq->dhchap_step)
253 			goto done_failure1;
254 		/* Validate negotiation parameters */
255 		status = nvmet_auth_negotiate(req, d);
256 		if (status == 0)
257 			req->sq->dhchap_step =
258 				NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
259 		else {
260 			req->sq->dhchap_step =
261 				NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
262 			req->sq->dhchap_status = status;
263 			status = 0;
264 		}
265 		goto done_kfree;
266 	}
267 	if (data->auth_id != req->sq->dhchap_step) {
268 		pr_debug("%s: ctrl %d qid %d step mismatch (%d != %d)\n",
269 			 __func__, ctrl->cntlid, req->sq->qid,
270 			 data->auth_id, req->sq->dhchap_step);
271 		goto done_failure1;
272 	}
273 	if (le16_to_cpu(data->t_id) != req->sq->dhchap_tid) {
274 		pr_debug("%s: ctrl %d qid %d invalid transaction %d (expected %d)\n",
275 			 __func__, ctrl->cntlid, req->sq->qid,
276 			 le16_to_cpu(data->t_id),
277 			 req->sq->dhchap_tid);
278 		req->sq->dhchap_step =
279 			NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
280 		req->sq->dhchap_status =
281 			NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
282 		goto done_kfree;
283 	}
284 
285 	switch (data->auth_id) {
286 	case NVME_AUTH_DHCHAP_MESSAGE_REPLY:
287 		status = nvmet_auth_reply(req, d);
288 		if (status == 0)
289 			req->sq->dhchap_step =
290 				NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
291 		else {
292 			req->sq->dhchap_step =
293 				NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
294 			req->sq->dhchap_status = status;
295 			status = 0;
296 		}
297 		goto done_kfree;
298 		break;
299 	case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2:
300 		req->sq->authenticated = true;
301 		pr_debug("%s: ctrl %d qid %d ctrl authenticated\n",
302 			 __func__, ctrl->cntlid, req->sq->qid);
303 		goto done_kfree;
304 		break;
305 	case NVME_AUTH_DHCHAP_MESSAGE_FAILURE2:
306 		status = nvmet_auth_failure2(d);
307 		if (status) {
308 			pr_warn("ctrl %d qid %d: authentication failed (%d)\n",
309 				ctrl->cntlid, req->sq->qid, status);
310 			req->sq->dhchap_status = status;
311 			req->sq->authenticated = false;
312 			status = 0;
313 		}
314 		goto done_kfree;
315 		break;
316 	default:
317 		req->sq->dhchap_status =
318 			NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
319 		req->sq->dhchap_step =
320 			NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
321 		req->sq->authenticated = false;
322 		goto done_kfree;
323 		break;
324 	}
325 done_failure1:
326 	req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
327 	req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
328 
329 done_kfree:
330 	kfree(d);
331 done:
332 	pr_debug("%s: ctrl %d qid %d dhchap status %x step %x\n", __func__,
333 		 ctrl->cntlid, req->sq->qid,
334 		 req->sq->dhchap_status, req->sq->dhchap_step);
335 	if (status)
336 		pr_debug("%s: ctrl %d qid %d nvme status %x error loc %d\n",
337 			 __func__, ctrl->cntlid, req->sq->qid,
338 			 status, req->error_loc);
339 	req->cqe->result.u64 = 0;
340 	nvmet_req_complete(req, status);
341 	if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
342 	    req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
343 		unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
344 
345 		mod_delayed_work(system_wq, &req->sq->auth_expired_work,
346 				 auth_expire_secs * HZ);
347 		return;
348 	}
349 	/* Final states, clear up variables */
350 	nvmet_auth_sq_free(req->sq);
351 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
352 		nvmet_ctrl_fatal_error(ctrl);
353 }
354 
nvmet_auth_challenge(struct nvmet_req * req,void * d,int al)355 static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
356 {
357 	struct nvmf_auth_dhchap_challenge_data *data = d;
358 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
359 	int ret = 0;
360 	int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
361 	int data_size = sizeof(*d) + hash_len;
362 
363 	if (ctrl->dh_tfm)
364 		data_size += ctrl->dh_keysize;
365 	if (al < data_size) {
366 		pr_debug("%s: buffer too small (al %d need %d)\n", __func__,
367 			 al, data_size);
368 		return -EINVAL;
369 	}
370 	memset(data, 0, data_size);
371 	req->sq->dhchap_s1 = nvme_auth_get_seqnum();
372 	data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
373 	data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
374 	data->t_id = cpu_to_le16(req->sq->dhchap_tid);
375 	data->hashid = ctrl->shash_id;
376 	data->hl = hash_len;
377 	data->seqnum = cpu_to_le32(req->sq->dhchap_s1);
378 	req->sq->dhchap_c1 = kmalloc(data->hl, GFP_KERNEL);
379 	if (!req->sq->dhchap_c1)
380 		return -ENOMEM;
381 	get_random_bytes(req->sq->dhchap_c1, data->hl);
382 	memcpy(data->cval, req->sq->dhchap_c1, data->hl);
383 	if (ctrl->dh_tfm) {
384 		data->dhgid = ctrl->dh_gid;
385 		data->dhvlen = cpu_to_le16(ctrl->dh_keysize);
386 		ret = nvmet_auth_ctrl_exponential(req, data->cval + data->hl,
387 						  ctrl->dh_keysize);
388 	}
389 	pr_debug("%s: ctrl %d qid %d seq %d transaction %d hl %d dhvlen %zu\n",
390 		 __func__, ctrl->cntlid, req->sq->qid, req->sq->dhchap_s1,
391 		 req->sq->dhchap_tid, data->hl, ctrl->dh_keysize);
392 	return ret;
393 }
394 
nvmet_auth_success1(struct nvmet_req * req,void * d,int al)395 static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al)
396 {
397 	struct nvmf_auth_dhchap_success1_data *data = d;
398 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
399 	int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
400 
401 	WARN_ON(al < sizeof(*data));
402 	memset(data, 0, sizeof(*data));
403 	data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
404 	data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
405 	data->t_id = cpu_to_le16(req->sq->dhchap_tid);
406 	data->hl = hash_len;
407 	if (req->sq->dhchap_c2) {
408 		if (!ctrl->ctrl_key) {
409 			pr_warn("ctrl %d qid %d no ctrl key\n",
410 				ctrl->cntlid, req->sq->qid);
411 			return NVME_AUTH_DHCHAP_FAILURE_FAILED;
412 		}
413 		if (nvmet_auth_ctrl_hash(req, data->rval, data->hl))
414 			return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
415 		data->rvalid = 1;
416 		pr_debug("ctrl %d qid %d response %*ph\n",
417 			 ctrl->cntlid, req->sq->qid, data->hl, data->rval);
418 	}
419 	return 0;
420 }
421 
nvmet_auth_failure1(struct nvmet_req * req,void * d,int al)422 static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al)
423 {
424 	struct nvmf_auth_dhchap_failure_data *data = d;
425 
426 	WARN_ON(al < sizeof(*data));
427 	data->auth_type = NVME_AUTH_COMMON_MESSAGES;
428 	data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
429 	data->t_id = cpu_to_le16(req->sq->dhchap_tid);
430 	data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
431 	data->rescode_exp = req->sq->dhchap_status;
432 }
433 
nvmet_execute_auth_receive(struct nvmet_req * req)434 void nvmet_execute_auth_receive(struct nvmet_req *req)
435 {
436 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
437 	void *d;
438 	u32 al;
439 	u16 status = 0;
440 
441 	if (req->cmd->auth_receive.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
442 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
443 		req->error_loc =
444 			offsetof(struct nvmf_auth_receive_command, secp);
445 		goto done;
446 	}
447 	if (req->cmd->auth_receive.spsp0 != 0x01) {
448 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
449 		req->error_loc =
450 			offsetof(struct nvmf_auth_receive_command, spsp0);
451 		goto done;
452 	}
453 	if (req->cmd->auth_receive.spsp1 != 0x01) {
454 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
455 		req->error_loc =
456 			offsetof(struct nvmf_auth_receive_command, spsp1);
457 		goto done;
458 	}
459 	al = le32_to_cpu(req->cmd->auth_receive.al);
460 	if (!al) {
461 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
462 		req->error_loc =
463 			offsetof(struct nvmf_auth_receive_command, al);
464 		goto done;
465 	}
466 	if (!nvmet_check_transfer_len(req, al)) {
467 		pr_debug("%s: transfer length mismatch (%u)\n", __func__, al);
468 		return;
469 	}
470 
471 	d = kmalloc(al, GFP_KERNEL);
472 	if (!d) {
473 		status = NVME_SC_INTERNAL;
474 		goto done;
475 	}
476 	pr_debug("%s: ctrl %d qid %d step %x\n", __func__,
477 		 ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
478 	switch (req->sq->dhchap_step) {
479 	case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
480 		if (nvmet_auth_challenge(req, d, al) < 0) {
481 			pr_warn("ctrl %d qid %d: challenge error (%d)\n",
482 				ctrl->cntlid, req->sq->qid, status);
483 			status = NVME_SC_INTERNAL;
484 			break;
485 		}
486 		if (status) {
487 			req->sq->dhchap_status = status;
488 			nvmet_auth_failure1(req, d, al);
489 			pr_warn("ctrl %d qid %d: challenge status (%x)\n",
490 				ctrl->cntlid, req->sq->qid,
491 				req->sq->dhchap_status);
492 			status = 0;
493 			break;
494 		}
495 		req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
496 		break;
497 	case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1:
498 		status = nvmet_auth_success1(req, d, al);
499 		if (status) {
500 			req->sq->dhchap_status = status;
501 			req->sq->authenticated = false;
502 			nvmet_auth_failure1(req, d, al);
503 			pr_warn("ctrl %d qid %d: success1 status (%x)\n",
504 				ctrl->cntlid, req->sq->qid,
505 				req->sq->dhchap_status);
506 			break;
507 		}
508 		req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
509 		break;
510 	case NVME_AUTH_DHCHAP_MESSAGE_FAILURE1:
511 		req->sq->authenticated = false;
512 		nvmet_auth_failure1(req, d, al);
513 		pr_warn("ctrl %d qid %d failure1 (%x)\n",
514 			ctrl->cntlid, req->sq->qid, req->sq->dhchap_status);
515 		break;
516 	default:
517 		pr_warn("ctrl %d qid %d unhandled step (%d)\n",
518 			ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
519 		req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
520 		req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
521 		nvmet_auth_failure1(req, d, al);
522 		status = 0;
523 		break;
524 	}
525 
526 	status = nvmet_copy_to_sgl(req, 0, d, al);
527 	kfree(d);
528 done:
529 	req->cqe->result.u64 = 0;
530 	nvmet_req_complete(req, status);
531 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
532 		nvmet_auth_sq_free(req->sq);
533 	else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
534 		nvmet_auth_sq_free(req->sq);
535 		nvmet_ctrl_fatal_error(ctrl);
536 	}
537 }
538