1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVM Express device driver
4 * Copyright (c) 2011-2014, Intel Corporation.
5 */
6
7 #include <linux/blkdev.h>
8 #include <linux/blk-mq.h>
9 #include <linux/blk-integrity.h>
10 #include <linux/compat.h>
11 #include <linux/delay.h>
12 #include <linux/errno.h>
13 #include <linux/hdreg.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/backing-dev.h>
17 #include <linux/slab.h>
18 #include <linux/types.h>
19 #include <linux/pr.h>
20 #include <linux/ptrace.h>
21 #include <linux/nvme_ioctl.h>
22 #include <linux/pm_qos.h>
23 #include <asm/unaligned.h>
24
25 #include "nvme.h"
26 #include "fabrics.h"
27 #include <linux/nvme-auth.h>
28
29 #define CREATE_TRACE_POINTS
30 #include "trace.h"
31
32 #define NVME_MINORS (1U << MINORBITS)
33
34 struct nvme_ns_info {
35 struct nvme_ns_ids ids;
36 u32 nsid;
37 __le32 anagrpid;
38 bool is_shared;
39 bool is_readonly;
40 bool is_ready;
41 };
42
43 unsigned int admin_timeout = 60;
44 module_param(admin_timeout, uint, 0644);
45 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
46 EXPORT_SYMBOL_GPL(admin_timeout);
47
48 unsigned int nvme_io_timeout = 30;
49 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
50 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
51 EXPORT_SYMBOL_GPL(nvme_io_timeout);
52
53 static unsigned char shutdown_timeout = 5;
54 module_param(shutdown_timeout, byte, 0644);
55 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
56
57 static u8 nvme_max_retries = 5;
58 module_param_named(max_retries, nvme_max_retries, byte, 0644);
59 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
60
61 static unsigned long default_ps_max_latency_us = 100000;
62 module_param(default_ps_max_latency_us, ulong, 0644);
63 MODULE_PARM_DESC(default_ps_max_latency_us,
64 "max power saving latency for new devices; use PM QOS to change per device");
65
66 static bool force_apst;
67 module_param(force_apst, bool, 0644);
68 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
69
70 static unsigned long apst_primary_timeout_ms = 100;
71 module_param(apst_primary_timeout_ms, ulong, 0644);
72 MODULE_PARM_DESC(apst_primary_timeout_ms,
73 "primary APST timeout in ms");
74
75 static unsigned long apst_secondary_timeout_ms = 2000;
76 module_param(apst_secondary_timeout_ms, ulong, 0644);
77 MODULE_PARM_DESC(apst_secondary_timeout_ms,
78 "secondary APST timeout in ms");
79
80 static unsigned long apst_primary_latency_tol_us = 15000;
81 module_param(apst_primary_latency_tol_us, ulong, 0644);
82 MODULE_PARM_DESC(apst_primary_latency_tol_us,
83 "primary APST latency tolerance in us");
84
85 static unsigned long apst_secondary_latency_tol_us = 100000;
86 module_param(apst_secondary_latency_tol_us, ulong, 0644);
87 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
88 "secondary APST latency tolerance in us");
89
90 /*
91 * nvme_wq - hosts nvme related works that are not reset or delete
92 * nvme_reset_wq - hosts nvme reset works
93 * nvme_delete_wq - hosts nvme delete works
94 *
95 * nvme_wq will host works such as scan, aen handling, fw activation,
96 * keep-alive, periodic reconnects etc. nvme_reset_wq
97 * runs reset works which also flush works hosted on nvme_wq for
98 * serialization purposes. nvme_delete_wq host controller deletion
99 * works which flush reset works for serialization.
100 */
101 struct workqueue_struct *nvme_wq;
102 EXPORT_SYMBOL_GPL(nvme_wq);
103
104 struct workqueue_struct *nvme_reset_wq;
105 EXPORT_SYMBOL_GPL(nvme_reset_wq);
106
107 struct workqueue_struct *nvme_delete_wq;
108 EXPORT_SYMBOL_GPL(nvme_delete_wq);
109
110 static LIST_HEAD(nvme_subsystems);
111 static DEFINE_MUTEX(nvme_subsystems_lock);
112
113 static DEFINE_IDA(nvme_instance_ida);
114 static dev_t nvme_ctrl_base_chr_devt;
115 static struct class *nvme_class;
116 static struct class *nvme_subsys_class;
117
118 static DEFINE_IDA(nvme_ns_chr_minor_ida);
119 static dev_t nvme_ns_chr_devt;
120 static struct class *nvme_ns_chr_class;
121
122 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
123 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
124 unsigned nsid);
125 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
126 struct nvme_command *cmd);
127
nvme_queue_scan(struct nvme_ctrl * ctrl)128 void nvme_queue_scan(struct nvme_ctrl *ctrl)
129 {
130 /*
131 * Only new queue scan work when admin and IO queues are both alive
132 */
133 if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
134 queue_work(nvme_wq, &ctrl->scan_work);
135 }
136
137 /*
138 * Use this function to proceed with scheduling reset_work for a controller
139 * that had previously been set to the resetting state. This is intended for
140 * code paths that can't be interrupted by other reset attempts. A hot removal
141 * may prevent this from succeeding.
142 */
nvme_try_sched_reset(struct nvme_ctrl * ctrl)143 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
144 {
145 if (ctrl->state != NVME_CTRL_RESETTING)
146 return -EBUSY;
147 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
148 return -EBUSY;
149 return 0;
150 }
151 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
152
nvme_failfast_work(struct work_struct * work)153 static void nvme_failfast_work(struct work_struct *work)
154 {
155 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
156 struct nvme_ctrl, failfast_work);
157
158 if (ctrl->state != NVME_CTRL_CONNECTING)
159 return;
160
161 set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
162 dev_info(ctrl->device, "failfast expired\n");
163 nvme_kick_requeue_lists(ctrl);
164 }
165
nvme_start_failfast_work(struct nvme_ctrl * ctrl)166 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
167 {
168 if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
169 return;
170
171 schedule_delayed_work(&ctrl->failfast_work,
172 ctrl->opts->fast_io_fail_tmo * HZ);
173 }
174
nvme_stop_failfast_work(struct nvme_ctrl * ctrl)175 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
176 {
177 if (!ctrl->opts)
178 return;
179
180 cancel_delayed_work_sync(&ctrl->failfast_work);
181 clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
182 }
183
184
nvme_reset_ctrl(struct nvme_ctrl * ctrl)185 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
186 {
187 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
188 return -EBUSY;
189 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
190 return -EBUSY;
191 return 0;
192 }
193 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
194
nvme_reset_ctrl_sync(struct nvme_ctrl * ctrl)195 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
196 {
197 int ret;
198
199 ret = nvme_reset_ctrl(ctrl);
200 if (!ret) {
201 flush_work(&ctrl->reset_work);
202 if (ctrl->state != NVME_CTRL_LIVE)
203 ret = -ENETRESET;
204 }
205
206 return ret;
207 }
208
nvme_do_delete_ctrl(struct nvme_ctrl * ctrl)209 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
210 {
211 dev_info(ctrl->device,
212 "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
213
214 flush_work(&ctrl->reset_work);
215 nvme_stop_ctrl(ctrl);
216 nvme_remove_namespaces(ctrl);
217 ctrl->ops->delete_ctrl(ctrl);
218 nvme_uninit_ctrl(ctrl);
219 }
220
nvme_delete_ctrl_work(struct work_struct * work)221 static void nvme_delete_ctrl_work(struct work_struct *work)
222 {
223 struct nvme_ctrl *ctrl =
224 container_of(work, struct nvme_ctrl, delete_work);
225
226 nvme_do_delete_ctrl(ctrl);
227 }
228
nvme_delete_ctrl(struct nvme_ctrl * ctrl)229 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
230 {
231 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
232 return -EBUSY;
233 if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
234 return -EBUSY;
235 return 0;
236 }
237 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
238
nvme_delete_ctrl_sync(struct nvme_ctrl * ctrl)239 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
240 {
241 /*
242 * Keep a reference until nvme_do_delete_ctrl() complete,
243 * since ->delete_ctrl can free the controller.
244 */
245 nvme_get_ctrl(ctrl);
246 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
247 nvme_do_delete_ctrl(ctrl);
248 nvme_put_ctrl(ctrl);
249 }
250
nvme_error_status(u16 status)251 static blk_status_t nvme_error_status(u16 status)
252 {
253 switch (status & 0x7ff) {
254 case NVME_SC_SUCCESS:
255 return BLK_STS_OK;
256 case NVME_SC_CAP_EXCEEDED:
257 return BLK_STS_NOSPC;
258 case NVME_SC_LBA_RANGE:
259 case NVME_SC_CMD_INTERRUPTED:
260 case NVME_SC_NS_NOT_READY:
261 return BLK_STS_TARGET;
262 case NVME_SC_BAD_ATTRIBUTES:
263 case NVME_SC_ONCS_NOT_SUPPORTED:
264 case NVME_SC_INVALID_OPCODE:
265 case NVME_SC_INVALID_FIELD:
266 case NVME_SC_INVALID_NS:
267 return BLK_STS_NOTSUPP;
268 case NVME_SC_WRITE_FAULT:
269 case NVME_SC_READ_ERROR:
270 case NVME_SC_UNWRITTEN_BLOCK:
271 case NVME_SC_ACCESS_DENIED:
272 case NVME_SC_READ_ONLY:
273 case NVME_SC_COMPARE_FAILED:
274 return BLK_STS_MEDIUM;
275 case NVME_SC_GUARD_CHECK:
276 case NVME_SC_APPTAG_CHECK:
277 case NVME_SC_REFTAG_CHECK:
278 case NVME_SC_INVALID_PI:
279 return BLK_STS_PROTECTION;
280 case NVME_SC_RESERVATION_CONFLICT:
281 return BLK_STS_NEXUS;
282 case NVME_SC_HOST_PATH_ERROR:
283 return BLK_STS_TRANSPORT;
284 case NVME_SC_ZONE_TOO_MANY_ACTIVE:
285 return BLK_STS_ZONE_ACTIVE_RESOURCE;
286 case NVME_SC_ZONE_TOO_MANY_OPEN:
287 return BLK_STS_ZONE_OPEN_RESOURCE;
288 default:
289 return BLK_STS_IOERR;
290 }
291 }
292
nvme_retry_req(struct request * req)293 static void nvme_retry_req(struct request *req)
294 {
295 unsigned long delay = 0;
296 u16 crd;
297
298 /* The mask and shift result must be <= 3 */
299 crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
300 if (crd)
301 delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
302
303 nvme_req(req)->retries++;
304 blk_mq_requeue_request(req, false);
305 blk_mq_delay_kick_requeue_list(req->q, delay);
306 }
307
nvme_log_error(struct request * req)308 static void nvme_log_error(struct request *req)
309 {
310 struct nvme_ns *ns = req->q->queuedata;
311 struct nvme_request *nr = nvme_req(req);
312
313 if (ns) {
314 pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
315 ns->disk ? ns->disk->disk_name : "?",
316 nvme_get_opcode_str(nr->cmd->common.opcode),
317 nr->cmd->common.opcode,
318 (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)),
319 (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
320 nvme_get_error_status_str(nr->status),
321 nr->status >> 8 & 7, /* Status Code Type */
322 nr->status & 0xff, /* Status Code */
323 nr->status & NVME_SC_MORE ? "MORE " : "",
324 nr->status & NVME_SC_DNR ? "DNR " : "");
325 return;
326 }
327
328 pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
329 dev_name(nr->ctrl->device),
330 nvme_get_admin_opcode_str(nr->cmd->common.opcode),
331 nr->cmd->common.opcode,
332 nvme_get_error_status_str(nr->status),
333 nr->status >> 8 & 7, /* Status Code Type */
334 nr->status & 0xff, /* Status Code */
335 nr->status & NVME_SC_MORE ? "MORE " : "",
336 nr->status & NVME_SC_DNR ? "DNR " : "");
337 }
338
339 enum nvme_disposition {
340 COMPLETE,
341 RETRY,
342 FAILOVER,
343 AUTHENTICATE,
344 };
345
nvme_decide_disposition(struct request * req)346 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
347 {
348 if (likely(nvme_req(req)->status == 0))
349 return COMPLETE;
350
351 if ((nvme_req(req)->status & 0x7ff) == NVME_SC_AUTH_REQUIRED)
352 return AUTHENTICATE;
353
354 if (blk_noretry_request(req) ||
355 (nvme_req(req)->status & NVME_SC_DNR) ||
356 nvme_req(req)->retries >= nvme_max_retries)
357 return COMPLETE;
358
359 if (req->cmd_flags & REQ_NVME_MPATH) {
360 if (nvme_is_path_error(nvme_req(req)->status) ||
361 blk_queue_dying(req->q))
362 return FAILOVER;
363 } else {
364 if (blk_queue_dying(req->q))
365 return COMPLETE;
366 }
367
368 return RETRY;
369 }
370
nvme_end_req_zoned(struct request * req)371 static inline void nvme_end_req_zoned(struct request *req)
372 {
373 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
374 req_op(req) == REQ_OP_ZONE_APPEND)
375 req->__sector = nvme_lba_to_sect(req->q->queuedata,
376 le64_to_cpu(nvme_req(req)->result.u64));
377 }
378
nvme_end_req(struct request * req)379 static inline void nvme_end_req(struct request *req)
380 {
381 blk_status_t status = nvme_error_status(nvme_req(req)->status);
382
383 if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET)))
384 nvme_log_error(req);
385 nvme_end_req_zoned(req);
386 nvme_trace_bio_complete(req);
387 blk_mq_end_request(req, status);
388 }
389
nvme_complete_rq(struct request * req)390 void nvme_complete_rq(struct request *req)
391 {
392 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
393
394 trace_nvme_complete_rq(req);
395 nvme_cleanup_cmd(req);
396
397 if (ctrl->kas)
398 ctrl->comp_seen = true;
399
400 switch (nvme_decide_disposition(req)) {
401 case COMPLETE:
402 nvme_end_req(req);
403 return;
404 case RETRY:
405 nvme_retry_req(req);
406 return;
407 case FAILOVER:
408 nvme_failover_req(req);
409 return;
410 case AUTHENTICATE:
411 #ifdef CONFIG_NVME_AUTH
412 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
413 nvme_retry_req(req);
414 #else
415 nvme_end_req(req);
416 #endif
417 return;
418 }
419 }
420 EXPORT_SYMBOL_GPL(nvme_complete_rq);
421
nvme_complete_batch_req(struct request * req)422 void nvme_complete_batch_req(struct request *req)
423 {
424 trace_nvme_complete_rq(req);
425 nvme_cleanup_cmd(req);
426 nvme_end_req_zoned(req);
427 }
428 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
429
430 /*
431 * Called to unwind from ->queue_rq on a failed command submission so that the
432 * multipathing code gets called to potentially failover to another path.
433 * The caller needs to unwind all transport specific resource allocations and
434 * must return propagate the return value.
435 */
nvme_host_path_error(struct request * req)436 blk_status_t nvme_host_path_error(struct request *req)
437 {
438 nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
439 blk_mq_set_request_complete(req);
440 nvme_complete_rq(req);
441 return BLK_STS_OK;
442 }
443 EXPORT_SYMBOL_GPL(nvme_host_path_error);
444
nvme_cancel_request(struct request * req,void * data)445 bool nvme_cancel_request(struct request *req, void *data)
446 {
447 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
448 "Cancelling I/O %d", req->tag);
449
450 /* don't abort one completed request */
451 if (blk_mq_request_completed(req))
452 return true;
453
454 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
455 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
456 blk_mq_complete_request(req);
457 return true;
458 }
459 EXPORT_SYMBOL_GPL(nvme_cancel_request);
460
nvme_cancel_tagset(struct nvme_ctrl * ctrl)461 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
462 {
463 if (ctrl->tagset) {
464 blk_mq_tagset_busy_iter(ctrl->tagset,
465 nvme_cancel_request, ctrl);
466 blk_mq_tagset_wait_completed_request(ctrl->tagset);
467 }
468 }
469 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
470
nvme_cancel_admin_tagset(struct nvme_ctrl * ctrl)471 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
472 {
473 if (ctrl->admin_tagset) {
474 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
475 nvme_cancel_request, ctrl);
476 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
477 }
478 }
479 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
480
nvme_change_ctrl_state(struct nvme_ctrl * ctrl,enum nvme_ctrl_state new_state)481 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
482 enum nvme_ctrl_state new_state)
483 {
484 enum nvme_ctrl_state old_state;
485 unsigned long flags;
486 bool changed = false;
487
488 spin_lock_irqsave(&ctrl->lock, flags);
489
490 old_state = ctrl->state;
491 switch (new_state) {
492 case NVME_CTRL_LIVE:
493 switch (old_state) {
494 case NVME_CTRL_NEW:
495 case NVME_CTRL_RESETTING:
496 case NVME_CTRL_CONNECTING:
497 changed = true;
498 fallthrough;
499 default:
500 break;
501 }
502 break;
503 case NVME_CTRL_RESETTING:
504 switch (old_state) {
505 case NVME_CTRL_NEW:
506 case NVME_CTRL_LIVE:
507 changed = true;
508 fallthrough;
509 default:
510 break;
511 }
512 break;
513 case NVME_CTRL_CONNECTING:
514 switch (old_state) {
515 case NVME_CTRL_NEW:
516 case NVME_CTRL_RESETTING:
517 changed = true;
518 fallthrough;
519 default:
520 break;
521 }
522 break;
523 case NVME_CTRL_DELETING:
524 switch (old_state) {
525 case NVME_CTRL_LIVE:
526 case NVME_CTRL_RESETTING:
527 case NVME_CTRL_CONNECTING:
528 changed = true;
529 fallthrough;
530 default:
531 break;
532 }
533 break;
534 case NVME_CTRL_DELETING_NOIO:
535 switch (old_state) {
536 case NVME_CTRL_DELETING:
537 case NVME_CTRL_DEAD:
538 changed = true;
539 fallthrough;
540 default:
541 break;
542 }
543 break;
544 case NVME_CTRL_DEAD:
545 switch (old_state) {
546 case NVME_CTRL_DELETING:
547 changed = true;
548 fallthrough;
549 default:
550 break;
551 }
552 break;
553 default:
554 break;
555 }
556
557 if (changed) {
558 ctrl->state = new_state;
559 wake_up_all(&ctrl->state_wq);
560 }
561
562 spin_unlock_irqrestore(&ctrl->lock, flags);
563 if (!changed)
564 return false;
565
566 if (ctrl->state == NVME_CTRL_LIVE) {
567 if (old_state == NVME_CTRL_CONNECTING)
568 nvme_stop_failfast_work(ctrl);
569 nvme_kick_requeue_lists(ctrl);
570 } else if (ctrl->state == NVME_CTRL_CONNECTING &&
571 old_state == NVME_CTRL_RESETTING) {
572 nvme_start_failfast_work(ctrl);
573 }
574 return changed;
575 }
576 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
577
578 /*
579 * Returns true for sink states that can't ever transition back to live.
580 */
nvme_state_terminal(struct nvme_ctrl * ctrl)581 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
582 {
583 switch (ctrl->state) {
584 case NVME_CTRL_NEW:
585 case NVME_CTRL_LIVE:
586 case NVME_CTRL_RESETTING:
587 case NVME_CTRL_CONNECTING:
588 return false;
589 case NVME_CTRL_DELETING:
590 case NVME_CTRL_DELETING_NOIO:
591 case NVME_CTRL_DEAD:
592 return true;
593 default:
594 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
595 return true;
596 }
597 }
598
599 /*
600 * Waits for the controller state to be resetting, or returns false if it is
601 * not possible to ever transition to that state.
602 */
nvme_wait_reset(struct nvme_ctrl * ctrl)603 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
604 {
605 wait_event(ctrl->state_wq,
606 nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
607 nvme_state_terminal(ctrl));
608 return ctrl->state == NVME_CTRL_RESETTING;
609 }
610 EXPORT_SYMBOL_GPL(nvme_wait_reset);
611
nvme_free_ns_head(struct kref * ref)612 static void nvme_free_ns_head(struct kref *ref)
613 {
614 struct nvme_ns_head *head =
615 container_of(ref, struct nvme_ns_head, ref);
616
617 nvme_mpath_remove_disk(head);
618 ida_free(&head->subsys->ns_ida, head->instance);
619 cleanup_srcu_struct(&head->srcu);
620 nvme_put_subsystem(head->subsys);
621 kfree(head);
622 }
623
nvme_tryget_ns_head(struct nvme_ns_head * head)624 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
625 {
626 return kref_get_unless_zero(&head->ref);
627 }
628
nvme_put_ns_head(struct nvme_ns_head * head)629 void nvme_put_ns_head(struct nvme_ns_head *head)
630 {
631 kref_put(&head->ref, nvme_free_ns_head);
632 }
633
nvme_free_ns(struct kref * kref)634 static void nvme_free_ns(struct kref *kref)
635 {
636 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
637
638 put_disk(ns->disk);
639 nvme_put_ns_head(ns->head);
640 nvme_put_ctrl(ns->ctrl);
641 kfree(ns);
642 }
643
nvme_get_ns(struct nvme_ns * ns)644 static inline bool nvme_get_ns(struct nvme_ns *ns)
645 {
646 return kref_get_unless_zero(&ns->kref);
647 }
648
nvme_put_ns(struct nvme_ns * ns)649 void nvme_put_ns(struct nvme_ns *ns)
650 {
651 kref_put(&ns->kref, nvme_free_ns);
652 }
653 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
654
nvme_clear_nvme_request(struct request * req)655 static inline void nvme_clear_nvme_request(struct request *req)
656 {
657 nvme_req(req)->status = 0;
658 nvme_req(req)->retries = 0;
659 nvme_req(req)->flags = 0;
660 req->rq_flags |= RQF_DONTPREP;
661 }
662
663 /* initialize a passthrough request */
nvme_init_request(struct request * req,struct nvme_command * cmd)664 void nvme_init_request(struct request *req, struct nvme_command *cmd)
665 {
666 if (req->q->queuedata)
667 req->timeout = NVME_IO_TIMEOUT;
668 else /* no queuedata implies admin queue */
669 req->timeout = NVME_ADMIN_TIMEOUT;
670
671 /* passthru commands should let the driver set the SGL flags */
672 cmd->common.flags &= ~NVME_CMD_SGL_ALL;
673
674 req->cmd_flags |= REQ_FAILFAST_DRIVER;
675 if (req->mq_hctx->type == HCTX_TYPE_POLL)
676 req->cmd_flags |= REQ_POLLED;
677 nvme_clear_nvme_request(req);
678 req->rq_flags |= RQF_QUIET;
679 memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
680 }
681 EXPORT_SYMBOL_GPL(nvme_init_request);
682
683 /*
684 * For something we're not in a state to send to the device the default action
685 * is to busy it and retry it after the controller state is recovered. However,
686 * if the controller is deleting or if anything is marked for failfast or
687 * nvme multipath it is immediately failed.
688 *
689 * Note: commands used to initialize the controller will be marked for failfast.
690 * Note: nvme cli/ioctl commands are marked for failfast.
691 */
nvme_fail_nonready_command(struct nvme_ctrl * ctrl,struct request * rq)692 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
693 struct request *rq)
694 {
695 if (ctrl->state != NVME_CTRL_DELETING_NOIO &&
696 ctrl->state != NVME_CTRL_DELETING &&
697 ctrl->state != NVME_CTRL_DEAD &&
698 !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
699 !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
700 return BLK_STS_RESOURCE;
701 return nvme_host_path_error(rq);
702 }
703 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
704
__nvme_check_ready(struct nvme_ctrl * ctrl,struct request * rq,bool queue_live)705 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
706 bool queue_live)
707 {
708 struct nvme_request *req = nvme_req(rq);
709
710 /*
711 * currently we have a problem sending passthru commands
712 * on the admin_q if the controller is not LIVE because we can't
713 * make sure that they are going out after the admin connect,
714 * controller enable and/or other commands in the initialization
715 * sequence. until the controller will be LIVE, fail with
716 * BLK_STS_RESOURCE so that they will be rescheduled.
717 */
718 if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
719 return false;
720
721 if (ctrl->ops->flags & NVME_F_FABRICS) {
722 /*
723 * Only allow commands on a live queue, except for the connect
724 * command, which is require to set the queue live in the
725 * appropinquate states.
726 */
727 switch (ctrl->state) {
728 case NVME_CTRL_CONNECTING:
729 if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
730 (req->cmd->fabrics.fctype == nvme_fabrics_type_connect ||
731 req->cmd->fabrics.fctype == nvme_fabrics_type_auth_send ||
732 req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive))
733 return true;
734 break;
735 default:
736 break;
737 case NVME_CTRL_DEAD:
738 return false;
739 }
740 }
741
742 return queue_live;
743 }
744 EXPORT_SYMBOL_GPL(__nvme_check_ready);
745
nvme_setup_flush(struct nvme_ns * ns,struct nvme_command * cmnd)746 static inline void nvme_setup_flush(struct nvme_ns *ns,
747 struct nvme_command *cmnd)
748 {
749 memset(cmnd, 0, sizeof(*cmnd));
750 cmnd->common.opcode = nvme_cmd_flush;
751 cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
752 }
753
nvme_setup_discard(struct nvme_ns * ns,struct request * req,struct nvme_command * cmnd)754 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
755 struct nvme_command *cmnd)
756 {
757 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
758 struct nvme_dsm_range *range;
759 struct bio *bio;
760
761 /*
762 * Some devices do not consider the DSM 'Number of Ranges' field when
763 * determining how much data to DMA. Always allocate memory for maximum
764 * number of segments to prevent device reading beyond end of buffer.
765 */
766 static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
767
768 range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
769 if (!range) {
770 /*
771 * If we fail allocation our range, fallback to the controller
772 * discard page. If that's also busy, it's safe to return
773 * busy, as we know we can make progress once that's freed.
774 */
775 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
776 return BLK_STS_RESOURCE;
777
778 range = page_address(ns->ctrl->discard_page);
779 }
780
781 __rq_for_each_bio(bio, req) {
782 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
783 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
784
785 if (n < segments) {
786 range[n].cattr = cpu_to_le32(0);
787 range[n].nlb = cpu_to_le32(nlb);
788 range[n].slba = cpu_to_le64(slba);
789 }
790 n++;
791 }
792
793 if (WARN_ON_ONCE(n != segments)) {
794 if (virt_to_page(range) == ns->ctrl->discard_page)
795 clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
796 else
797 kfree(range);
798 return BLK_STS_IOERR;
799 }
800
801 memset(cmnd, 0, sizeof(*cmnd));
802 cmnd->dsm.opcode = nvme_cmd_dsm;
803 cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
804 cmnd->dsm.nr = cpu_to_le32(segments - 1);
805 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
806
807 req->special_vec.bv_page = virt_to_page(range);
808 req->special_vec.bv_offset = offset_in_page(range);
809 req->special_vec.bv_len = alloc_size;
810 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
811
812 return BLK_STS_OK;
813 }
814
nvme_set_ref_tag(struct nvme_ns * ns,struct nvme_command * cmnd,struct request * req)815 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
816 struct request *req)
817 {
818 u32 upper, lower;
819 u64 ref48;
820
821 /* both rw and write zeroes share the same reftag format */
822 switch (ns->guard_type) {
823 case NVME_NVM_NS_16B_GUARD:
824 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
825 break;
826 case NVME_NVM_NS_64B_GUARD:
827 ref48 = ext_pi_ref_tag(req);
828 lower = lower_32_bits(ref48);
829 upper = upper_32_bits(ref48);
830
831 cmnd->rw.reftag = cpu_to_le32(lower);
832 cmnd->rw.cdw3 = cpu_to_le32(upper);
833 break;
834 default:
835 break;
836 }
837 }
838
nvme_setup_write_zeroes(struct nvme_ns * ns,struct request * req,struct nvme_command * cmnd)839 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
840 struct request *req, struct nvme_command *cmnd)
841 {
842 memset(cmnd, 0, sizeof(*cmnd));
843
844 if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
845 return nvme_setup_discard(ns, req, cmnd);
846
847 cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
848 cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
849 cmnd->write_zeroes.slba =
850 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
851 cmnd->write_zeroes.length =
852 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
853
854 if (nvme_ns_has_pi(ns)) {
855 cmnd->write_zeroes.control = cpu_to_le16(NVME_RW_PRINFO_PRACT);
856
857 switch (ns->pi_type) {
858 case NVME_NS_DPS_PI_TYPE1:
859 case NVME_NS_DPS_PI_TYPE2:
860 nvme_set_ref_tag(ns, cmnd, req);
861 break;
862 }
863 }
864
865 return BLK_STS_OK;
866 }
867
nvme_setup_rw(struct nvme_ns * ns,struct request * req,struct nvme_command * cmnd,enum nvme_opcode op)868 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
869 struct request *req, struct nvme_command *cmnd,
870 enum nvme_opcode op)
871 {
872 u16 control = 0;
873 u32 dsmgmt = 0;
874
875 if (req->cmd_flags & REQ_FUA)
876 control |= NVME_RW_FUA;
877 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
878 control |= NVME_RW_LR;
879
880 if (req->cmd_flags & REQ_RAHEAD)
881 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
882
883 cmnd->rw.opcode = op;
884 cmnd->rw.flags = 0;
885 cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
886 cmnd->rw.cdw2 = 0;
887 cmnd->rw.cdw3 = 0;
888 cmnd->rw.metadata = 0;
889 cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
890 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
891 cmnd->rw.reftag = 0;
892 cmnd->rw.apptag = 0;
893 cmnd->rw.appmask = 0;
894
895 if (ns->ms) {
896 /*
897 * If formated with metadata, the block layer always provides a
898 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else
899 * we enable the PRACT bit for protection information or set the
900 * namespace capacity to zero to prevent any I/O.
901 */
902 if (!blk_integrity_rq(req)) {
903 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
904 return BLK_STS_NOTSUPP;
905 control |= NVME_RW_PRINFO_PRACT;
906 }
907
908 switch (ns->pi_type) {
909 case NVME_NS_DPS_PI_TYPE3:
910 control |= NVME_RW_PRINFO_PRCHK_GUARD;
911 break;
912 case NVME_NS_DPS_PI_TYPE1:
913 case NVME_NS_DPS_PI_TYPE2:
914 control |= NVME_RW_PRINFO_PRCHK_GUARD |
915 NVME_RW_PRINFO_PRCHK_REF;
916 if (op == nvme_cmd_zone_append)
917 control |= NVME_RW_APPEND_PIREMAP;
918 nvme_set_ref_tag(ns, cmnd, req);
919 break;
920 }
921 }
922
923 cmnd->rw.control = cpu_to_le16(control);
924 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
925 return 0;
926 }
927
nvme_cleanup_cmd(struct request * req)928 void nvme_cleanup_cmd(struct request *req)
929 {
930 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
931 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
932
933 if (req->special_vec.bv_page == ctrl->discard_page)
934 clear_bit_unlock(0, &ctrl->discard_page_busy);
935 else
936 kfree(bvec_virt(&req->special_vec));
937 }
938 }
939 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
940
nvme_setup_cmd(struct nvme_ns * ns,struct request * req)941 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
942 {
943 struct nvme_command *cmd = nvme_req(req)->cmd;
944 blk_status_t ret = BLK_STS_OK;
945
946 if (!(req->rq_flags & RQF_DONTPREP))
947 nvme_clear_nvme_request(req);
948
949 switch (req_op(req)) {
950 case REQ_OP_DRV_IN:
951 case REQ_OP_DRV_OUT:
952 /* these are setup prior to execution in nvme_init_request() */
953 break;
954 case REQ_OP_FLUSH:
955 nvme_setup_flush(ns, cmd);
956 break;
957 case REQ_OP_ZONE_RESET_ALL:
958 case REQ_OP_ZONE_RESET:
959 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
960 break;
961 case REQ_OP_ZONE_OPEN:
962 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
963 break;
964 case REQ_OP_ZONE_CLOSE:
965 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
966 break;
967 case REQ_OP_ZONE_FINISH:
968 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
969 break;
970 case REQ_OP_WRITE_ZEROES:
971 ret = nvme_setup_write_zeroes(ns, req, cmd);
972 break;
973 case REQ_OP_DISCARD:
974 ret = nvme_setup_discard(ns, req, cmd);
975 break;
976 case REQ_OP_READ:
977 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
978 break;
979 case REQ_OP_WRITE:
980 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
981 break;
982 case REQ_OP_ZONE_APPEND:
983 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
984 break;
985 default:
986 WARN_ON_ONCE(1);
987 return BLK_STS_IOERR;
988 }
989
990 cmd->common.command_id = nvme_cid(req);
991 trace_nvme_setup_cmd(req, cmd);
992 return ret;
993 }
994 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
995
996 /*
997 * Return values:
998 * 0: success
999 * >0: nvme controller's cqe status response
1000 * <0: kernel error in lieu of controller response
1001 */
nvme_execute_rq(struct request * rq,bool at_head)1002 static int nvme_execute_rq(struct request *rq, bool at_head)
1003 {
1004 blk_status_t status;
1005
1006 status = blk_execute_rq(rq, at_head);
1007 if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
1008 return -EINTR;
1009 if (nvme_req(rq)->status)
1010 return nvme_req(rq)->status;
1011 return blk_status_to_errno(status);
1012 }
1013
1014 /*
1015 * Returns 0 on success. If the result is negative, it's a Linux error code;
1016 * if the result is positive, it's an NVM Express status code
1017 */
__nvme_submit_sync_cmd(struct request_queue * q,struct nvme_command * cmd,union nvme_result * result,void * buffer,unsigned bufflen,int qid,int at_head,blk_mq_req_flags_t flags)1018 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1019 union nvme_result *result, void *buffer, unsigned bufflen,
1020 int qid, int at_head, blk_mq_req_flags_t flags)
1021 {
1022 struct request *req;
1023 int ret;
1024
1025 if (qid == NVME_QID_ANY)
1026 req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
1027 else
1028 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
1029 qid - 1);
1030
1031 if (IS_ERR(req))
1032 return PTR_ERR(req);
1033 nvme_init_request(req, cmd);
1034
1035 if (buffer && bufflen) {
1036 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1037 if (ret)
1038 goto out;
1039 }
1040
1041 ret = nvme_execute_rq(req, at_head);
1042 if (result && ret >= 0)
1043 *result = nvme_req(req)->result;
1044 out:
1045 blk_mq_free_request(req);
1046 return ret;
1047 }
1048 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1049
nvme_submit_sync_cmd(struct request_queue * q,struct nvme_command * cmd,void * buffer,unsigned bufflen)1050 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1051 void *buffer, unsigned bufflen)
1052 {
1053 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
1054 NVME_QID_ANY, 0, 0);
1055 }
1056 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1057
nvme_known_admin_effects(u8 opcode)1058 static u32 nvme_known_admin_effects(u8 opcode)
1059 {
1060 switch (opcode) {
1061 case nvme_admin_format_nvm:
1062 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC |
1063 NVME_CMD_EFFECTS_CSE_MASK;
1064 case nvme_admin_sanitize_nvm:
1065 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK;
1066 default:
1067 break;
1068 }
1069 return 0;
1070 }
1071
nvme_known_nvm_effects(u8 opcode)1072 static u32 nvme_known_nvm_effects(u8 opcode)
1073 {
1074 switch (opcode) {
1075 case nvme_cmd_write:
1076 case nvme_cmd_write_zeroes:
1077 case nvme_cmd_write_uncor:
1078 return NVME_CMD_EFFECTS_LBCC;
1079 default:
1080 return 0;
1081 }
1082 }
1083
nvme_command_effects(struct nvme_ctrl * ctrl,struct nvme_ns * ns,u8 opcode)1084 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1085 {
1086 u32 effects = 0;
1087
1088 if (ns) {
1089 if (ns->head->effects)
1090 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1091 if (ns->head->ids.csi == NVME_CSI_NVM)
1092 effects |= nvme_known_nvm_effects(opcode);
1093 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1094 dev_warn_once(ctrl->device,
1095 "IO command:%02x has unusual effects:%08x\n",
1096 opcode, effects);
1097
1098 /*
1099 * NVME_CMD_EFFECTS_CSE_MASK causes a freeze all I/O queues,
1100 * which would deadlock when done on an I/O command. Note that
1101 * We already warn about an unusual effect above.
1102 */
1103 effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1104 } else {
1105 if (ctrl->effects)
1106 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1107 effects |= nvme_known_admin_effects(opcode);
1108 }
1109
1110 return effects;
1111 }
1112 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1113
nvme_passthru_start(struct nvme_ctrl * ctrl,struct nvme_ns * ns,u8 opcode)1114 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1115 u8 opcode)
1116 {
1117 u32 effects = nvme_command_effects(ctrl, ns, opcode);
1118
1119 /*
1120 * For simplicity, IO to all namespaces is quiesced even if the command
1121 * effects say only one namespace is affected.
1122 */
1123 if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1124 mutex_lock(&ctrl->scan_lock);
1125 mutex_lock(&ctrl->subsys->lock);
1126 nvme_mpath_start_freeze(ctrl->subsys);
1127 nvme_mpath_wait_freeze(ctrl->subsys);
1128 nvme_start_freeze(ctrl);
1129 nvme_wait_freeze(ctrl);
1130 }
1131 return effects;
1132 }
1133
nvme_passthru_end(struct nvme_ctrl * ctrl,u32 effects,struct nvme_command * cmd,int status)1134 void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,
1135 struct nvme_command *cmd, int status)
1136 {
1137 if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1138 nvme_unfreeze(ctrl);
1139 nvme_mpath_unfreeze(ctrl->subsys);
1140 mutex_unlock(&ctrl->subsys->lock);
1141 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1142 mutex_unlock(&ctrl->scan_lock);
1143 }
1144 if (effects & NVME_CMD_EFFECTS_CCC)
1145 nvme_init_ctrl_finish(ctrl);
1146 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1147 nvme_queue_scan(ctrl);
1148 flush_work(&ctrl->scan_work);
1149 }
1150
1151 switch (cmd->common.opcode) {
1152 case nvme_admin_set_features:
1153 switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1154 case NVME_FEAT_KATO:
1155 /*
1156 * Keep alive commands interval on the host should be
1157 * updated when KATO is modified by Set Features
1158 * commands.
1159 */
1160 if (!status)
1161 nvme_update_keep_alive(ctrl, cmd);
1162 break;
1163 default:
1164 break;
1165 }
1166 break;
1167 default:
1168 break;
1169 }
1170 }
1171 EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU);
1172
nvme_execute_passthru_rq(struct request * rq,u32 * effects)1173 int nvme_execute_passthru_rq(struct request *rq, u32 *effects)
1174 {
1175 struct nvme_command *cmd = nvme_req(rq)->cmd;
1176 struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1177 struct nvme_ns *ns = rq->q->queuedata;
1178
1179 *effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1180 return nvme_execute_rq(rq, false);
1181 }
1182 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1183
1184 /*
1185 * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1186 *
1187 * The host should send Keep Alive commands at half of the Keep Alive Timeout
1188 * accounting for transport roundtrip times [..].
1189 */
nvme_queue_keep_alive_work(struct nvme_ctrl * ctrl)1190 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1191 {
1192 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2);
1193 }
1194
nvme_keep_alive_end_io(struct request * rq,blk_status_t status)1195 static enum rq_end_io_ret nvme_keep_alive_end_io(struct request *rq,
1196 blk_status_t status)
1197 {
1198 struct nvme_ctrl *ctrl = rq->end_io_data;
1199 unsigned long flags;
1200 bool startka = false;
1201
1202 blk_mq_free_request(rq);
1203
1204 if (status) {
1205 dev_err(ctrl->device,
1206 "failed nvme_keep_alive_end_io error=%d\n",
1207 status);
1208 return RQ_END_IO_NONE;
1209 }
1210
1211 ctrl->comp_seen = false;
1212 spin_lock_irqsave(&ctrl->lock, flags);
1213 if (ctrl->state == NVME_CTRL_LIVE ||
1214 ctrl->state == NVME_CTRL_CONNECTING)
1215 startka = true;
1216 spin_unlock_irqrestore(&ctrl->lock, flags);
1217 if (startka)
1218 nvme_queue_keep_alive_work(ctrl);
1219 return RQ_END_IO_NONE;
1220 }
1221
nvme_keep_alive_work(struct work_struct * work)1222 static void nvme_keep_alive_work(struct work_struct *work)
1223 {
1224 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1225 struct nvme_ctrl, ka_work);
1226 bool comp_seen = ctrl->comp_seen;
1227 struct request *rq;
1228
1229 if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1230 dev_dbg(ctrl->device,
1231 "reschedule traffic based keep-alive timer\n");
1232 ctrl->comp_seen = false;
1233 nvme_queue_keep_alive_work(ctrl);
1234 return;
1235 }
1236
1237 rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1238 BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1239 if (IS_ERR(rq)) {
1240 /* allocation failure, reset the controller */
1241 dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1242 nvme_reset_ctrl(ctrl);
1243 return;
1244 }
1245 nvme_init_request(rq, &ctrl->ka_cmd);
1246
1247 rq->timeout = ctrl->kato * HZ;
1248 rq->end_io = nvme_keep_alive_end_io;
1249 rq->end_io_data = ctrl;
1250 blk_execute_rq_nowait(rq, false);
1251 }
1252
nvme_start_keep_alive(struct nvme_ctrl * ctrl)1253 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1254 {
1255 if (unlikely(ctrl->kato == 0))
1256 return;
1257
1258 nvme_queue_keep_alive_work(ctrl);
1259 }
1260
nvme_stop_keep_alive(struct nvme_ctrl * ctrl)1261 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1262 {
1263 if (unlikely(ctrl->kato == 0))
1264 return;
1265
1266 cancel_delayed_work_sync(&ctrl->ka_work);
1267 }
1268 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1269
nvme_update_keep_alive(struct nvme_ctrl * ctrl,struct nvme_command * cmd)1270 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1271 struct nvme_command *cmd)
1272 {
1273 unsigned int new_kato =
1274 DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1275
1276 dev_info(ctrl->device,
1277 "keep alive interval updated from %u ms to %u ms\n",
1278 ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1279
1280 nvme_stop_keep_alive(ctrl);
1281 ctrl->kato = new_kato;
1282 nvme_start_keep_alive(ctrl);
1283 }
1284
1285 /*
1286 * In NVMe 1.0 the CNS field was just a binary controller or namespace
1287 * flag, thus sending any new CNS opcodes has a big chance of not working.
1288 * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1289 * (but not for any later version).
1290 */
nvme_ctrl_limited_cns(struct nvme_ctrl * ctrl)1291 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1292 {
1293 if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1294 return ctrl->vs < NVME_VS(1, 2, 0);
1295 return ctrl->vs < NVME_VS(1, 1, 0);
1296 }
1297
nvme_identify_ctrl(struct nvme_ctrl * dev,struct nvme_id_ctrl ** id)1298 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1299 {
1300 struct nvme_command c = { };
1301 int error;
1302
1303 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1304 c.identify.opcode = nvme_admin_identify;
1305 c.identify.cns = NVME_ID_CNS_CTRL;
1306
1307 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1308 if (!*id)
1309 return -ENOMEM;
1310
1311 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1312 sizeof(struct nvme_id_ctrl));
1313 if (error)
1314 kfree(*id);
1315 return error;
1316 }
1317
nvme_process_ns_desc(struct nvme_ctrl * ctrl,struct nvme_ns_ids * ids,struct nvme_ns_id_desc * cur,bool * csi_seen)1318 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1319 struct nvme_ns_id_desc *cur, bool *csi_seen)
1320 {
1321 const char *warn_str = "ctrl returned bogus length:";
1322 void *data = cur;
1323
1324 switch (cur->nidt) {
1325 case NVME_NIDT_EUI64:
1326 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1327 dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1328 warn_str, cur->nidl);
1329 return -1;
1330 }
1331 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1332 return NVME_NIDT_EUI64_LEN;
1333 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1334 return NVME_NIDT_EUI64_LEN;
1335 case NVME_NIDT_NGUID:
1336 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1337 dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1338 warn_str, cur->nidl);
1339 return -1;
1340 }
1341 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1342 return NVME_NIDT_NGUID_LEN;
1343 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1344 return NVME_NIDT_NGUID_LEN;
1345 case NVME_NIDT_UUID:
1346 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1347 dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1348 warn_str, cur->nidl);
1349 return -1;
1350 }
1351 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1352 return NVME_NIDT_UUID_LEN;
1353 uuid_copy(&ids->uuid, data + sizeof(*cur));
1354 return NVME_NIDT_UUID_LEN;
1355 case NVME_NIDT_CSI:
1356 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1357 dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1358 warn_str, cur->nidl);
1359 return -1;
1360 }
1361 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1362 *csi_seen = true;
1363 return NVME_NIDT_CSI_LEN;
1364 default:
1365 /* Skip unknown types */
1366 return cur->nidl;
1367 }
1368 }
1369
nvme_identify_ns_descs(struct nvme_ctrl * ctrl,struct nvme_ns_info * info)1370 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,
1371 struct nvme_ns_info *info)
1372 {
1373 struct nvme_command c = { };
1374 bool csi_seen = false;
1375 int status, pos, len;
1376 void *data;
1377
1378 if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1379 return 0;
1380 if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1381 return 0;
1382
1383 c.identify.opcode = nvme_admin_identify;
1384 c.identify.nsid = cpu_to_le32(info->nsid);
1385 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1386
1387 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1388 if (!data)
1389 return -ENOMEM;
1390
1391 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1392 NVME_IDENTIFY_DATA_SIZE);
1393 if (status) {
1394 dev_warn(ctrl->device,
1395 "Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1396 info->nsid, status);
1397 goto free_data;
1398 }
1399
1400 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1401 struct nvme_ns_id_desc *cur = data + pos;
1402
1403 if (cur->nidl == 0)
1404 break;
1405
1406 len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen);
1407 if (len < 0)
1408 break;
1409
1410 len += sizeof(*cur);
1411 }
1412
1413 if (nvme_multi_css(ctrl) && !csi_seen) {
1414 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1415 info->nsid);
1416 status = -EINVAL;
1417 }
1418
1419 free_data:
1420 kfree(data);
1421 return status;
1422 }
1423
nvme_identify_ns(struct nvme_ctrl * ctrl,unsigned nsid,struct nvme_id_ns ** id)1424 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1425 struct nvme_id_ns **id)
1426 {
1427 struct nvme_command c = { };
1428 int error;
1429
1430 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1431 c.identify.opcode = nvme_admin_identify;
1432 c.identify.nsid = cpu_to_le32(nsid);
1433 c.identify.cns = NVME_ID_CNS_NS;
1434
1435 *id = kmalloc(sizeof(**id), GFP_KERNEL);
1436 if (!*id)
1437 return -ENOMEM;
1438
1439 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1440 if (error) {
1441 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1442 goto out_free_id;
1443 }
1444
1445 error = NVME_SC_INVALID_NS | NVME_SC_DNR;
1446 if ((*id)->ncap == 0) /* namespace not allocated or attached */
1447 goto out_free_id;
1448 return 0;
1449
1450 out_free_id:
1451 kfree(*id);
1452 return error;
1453 }
1454
nvme_ns_info_from_identify(struct nvme_ctrl * ctrl,struct nvme_ns_info * info)1455 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,
1456 struct nvme_ns_info *info)
1457 {
1458 struct nvme_ns_ids *ids = &info->ids;
1459 struct nvme_id_ns *id;
1460 int ret;
1461
1462 ret = nvme_identify_ns(ctrl, info->nsid, &id);
1463 if (ret)
1464 return ret;
1465 info->anagrpid = id->anagrpid;
1466 info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1467 info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1468 info->is_ready = true;
1469 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1470 dev_info(ctrl->device,
1471 "Ignoring bogus Namespace Identifiers\n");
1472 } else {
1473 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1474 !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1475 memcpy(ids->eui64, id->eui64, sizeof(ids->eui64));
1476 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1477 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1478 memcpy(ids->nguid, id->nguid, sizeof(ids->nguid));
1479 }
1480 kfree(id);
1481 return 0;
1482 }
1483
nvme_ns_info_from_id_cs_indep(struct nvme_ctrl * ctrl,struct nvme_ns_info * info)1484 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,
1485 struct nvme_ns_info *info)
1486 {
1487 struct nvme_id_ns_cs_indep *id;
1488 struct nvme_command c = {
1489 .identify.opcode = nvme_admin_identify,
1490 .identify.nsid = cpu_to_le32(info->nsid),
1491 .identify.cns = NVME_ID_CNS_NS_CS_INDEP,
1492 };
1493 int ret;
1494
1495 id = kmalloc(sizeof(*id), GFP_KERNEL);
1496 if (!id)
1497 return -ENOMEM;
1498
1499 ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1500 if (!ret) {
1501 info->anagrpid = id->anagrpid;
1502 info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1503 info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1504 info->is_ready = id->nstat & NVME_NSTAT_NRDY;
1505 }
1506 kfree(id);
1507 return ret;
1508 }
1509
nvme_features(struct nvme_ctrl * dev,u8 op,unsigned int fid,unsigned int dword11,void * buffer,size_t buflen,u32 * result)1510 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1511 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1512 {
1513 union nvme_result res = { 0 };
1514 struct nvme_command c = { };
1515 int ret;
1516
1517 c.features.opcode = op;
1518 c.features.fid = cpu_to_le32(fid);
1519 c.features.dword11 = cpu_to_le32(dword11);
1520
1521 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1522 buffer, buflen, NVME_QID_ANY, 0, 0);
1523 if (ret >= 0 && result)
1524 *result = le32_to_cpu(res.u32);
1525 return ret;
1526 }
1527
nvme_set_features(struct nvme_ctrl * dev,unsigned int fid,unsigned int dword11,void * buffer,size_t buflen,u32 * result)1528 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1529 unsigned int dword11, void *buffer, size_t buflen,
1530 u32 *result)
1531 {
1532 return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1533 buflen, result);
1534 }
1535 EXPORT_SYMBOL_GPL(nvme_set_features);
1536
nvme_get_features(struct nvme_ctrl * dev,unsigned int fid,unsigned int dword11,void * buffer,size_t buflen,u32 * result)1537 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1538 unsigned int dword11, void *buffer, size_t buflen,
1539 u32 *result)
1540 {
1541 return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1542 buflen, result);
1543 }
1544 EXPORT_SYMBOL_GPL(nvme_get_features);
1545
nvme_set_queue_count(struct nvme_ctrl * ctrl,int * count)1546 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1547 {
1548 u32 q_count = (*count - 1) | ((*count - 1) << 16);
1549 u32 result;
1550 int status, nr_io_queues;
1551
1552 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1553 &result);
1554 if (status < 0)
1555 return status;
1556
1557 /*
1558 * Degraded controllers might return an error when setting the queue
1559 * count. We still want to be able to bring them online and offer
1560 * access to the admin queue, as that might be only way to fix them up.
1561 */
1562 if (status > 0) {
1563 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1564 *count = 0;
1565 } else {
1566 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1567 *count = min(*count, nr_io_queues);
1568 }
1569
1570 return 0;
1571 }
1572 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1573
1574 #define NVME_AEN_SUPPORTED \
1575 (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1576 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1577
nvme_enable_aen(struct nvme_ctrl * ctrl)1578 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1579 {
1580 u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1581 int status;
1582
1583 if (!supported_aens)
1584 return;
1585
1586 status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1587 NULL, 0, &result);
1588 if (status)
1589 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1590 supported_aens);
1591
1592 queue_work(nvme_wq, &ctrl->async_event_work);
1593 }
1594
nvme_ns_open(struct nvme_ns * ns)1595 static int nvme_ns_open(struct nvme_ns *ns)
1596 {
1597
1598 /* should never be called due to GENHD_FL_HIDDEN */
1599 if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1600 goto fail;
1601 if (!nvme_get_ns(ns))
1602 goto fail;
1603 if (!try_module_get(ns->ctrl->ops->module))
1604 goto fail_put_ns;
1605
1606 return 0;
1607
1608 fail_put_ns:
1609 nvme_put_ns(ns);
1610 fail:
1611 return -ENXIO;
1612 }
1613
nvme_ns_release(struct nvme_ns * ns)1614 static void nvme_ns_release(struct nvme_ns *ns)
1615 {
1616
1617 module_put(ns->ctrl->ops->module);
1618 nvme_put_ns(ns);
1619 }
1620
nvme_open(struct block_device * bdev,fmode_t mode)1621 static int nvme_open(struct block_device *bdev, fmode_t mode)
1622 {
1623 return nvme_ns_open(bdev->bd_disk->private_data);
1624 }
1625
nvme_release(struct gendisk * disk,fmode_t mode)1626 static void nvme_release(struct gendisk *disk, fmode_t mode)
1627 {
1628 nvme_ns_release(disk->private_data);
1629 }
1630
nvme_getgeo(struct block_device * bdev,struct hd_geometry * geo)1631 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1632 {
1633 /* some standard values */
1634 geo->heads = 1 << 6;
1635 geo->sectors = 1 << 5;
1636 geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1637 return 0;
1638 }
1639
1640 #ifdef CONFIG_BLK_DEV_INTEGRITY
nvme_init_integrity(struct gendisk * disk,struct nvme_ns * ns,u32 max_integrity_segments)1641 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1642 u32 max_integrity_segments)
1643 {
1644 struct blk_integrity integrity = { };
1645
1646 switch (ns->pi_type) {
1647 case NVME_NS_DPS_PI_TYPE3:
1648 switch (ns->guard_type) {
1649 case NVME_NVM_NS_16B_GUARD:
1650 integrity.profile = &t10_pi_type3_crc;
1651 integrity.tag_size = sizeof(u16) + sizeof(u32);
1652 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1653 break;
1654 case NVME_NVM_NS_64B_GUARD:
1655 integrity.profile = &ext_pi_type3_crc64;
1656 integrity.tag_size = sizeof(u16) + 6;
1657 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1658 break;
1659 default:
1660 integrity.profile = NULL;
1661 break;
1662 }
1663 break;
1664 case NVME_NS_DPS_PI_TYPE1:
1665 case NVME_NS_DPS_PI_TYPE2:
1666 switch (ns->guard_type) {
1667 case NVME_NVM_NS_16B_GUARD:
1668 integrity.profile = &t10_pi_type1_crc;
1669 integrity.tag_size = sizeof(u16);
1670 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1671 break;
1672 case NVME_NVM_NS_64B_GUARD:
1673 integrity.profile = &ext_pi_type1_crc64;
1674 integrity.tag_size = sizeof(u16);
1675 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1676 break;
1677 default:
1678 integrity.profile = NULL;
1679 break;
1680 }
1681 break;
1682 default:
1683 integrity.profile = NULL;
1684 break;
1685 }
1686
1687 integrity.tuple_size = ns->ms;
1688 blk_integrity_register(disk, &integrity);
1689 blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1690 }
1691 #else
nvme_init_integrity(struct gendisk * disk,struct nvme_ns * ns,u32 max_integrity_segments)1692 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1693 u32 max_integrity_segments)
1694 {
1695 }
1696 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1697
nvme_config_discard(struct gendisk * disk,struct nvme_ns * ns)1698 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1699 {
1700 struct nvme_ctrl *ctrl = ns->ctrl;
1701 struct request_queue *queue = disk->queue;
1702 u32 size = queue_logical_block_size(queue);
1703
1704 if (ctrl->max_discard_sectors == 0) {
1705 blk_queue_max_discard_sectors(queue, 0);
1706 return;
1707 }
1708
1709 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1710 NVME_DSM_MAX_RANGES);
1711
1712 queue->limits.discard_granularity = size;
1713
1714 /* If discard is already enabled, don't reset queue limits */
1715 if (queue->limits.max_discard_sectors)
1716 return;
1717
1718 if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns, UINT_MAX))
1719 ctrl->max_discard_sectors = nvme_lba_to_sect(ns, ctrl->dmrsl);
1720
1721 blk_queue_max_discard_sectors(queue, ctrl->max_discard_sectors);
1722 blk_queue_max_discard_segments(queue, ctrl->max_discard_segments);
1723
1724 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1725 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1726 }
1727
nvme_ns_ids_equal(struct nvme_ns_ids * a,struct nvme_ns_ids * b)1728 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1729 {
1730 return uuid_equal(&a->uuid, &b->uuid) &&
1731 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1732 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1733 a->csi == b->csi;
1734 }
1735
nvme_init_ms(struct nvme_ns * ns,struct nvme_id_ns * id)1736 static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id)
1737 {
1738 bool first = id->dps & NVME_NS_DPS_PI_FIRST;
1739 unsigned lbaf = nvme_lbaf_index(id->flbas);
1740 struct nvme_ctrl *ctrl = ns->ctrl;
1741 struct nvme_command c = { };
1742 struct nvme_id_ns_nvm *nvm;
1743 int ret = 0;
1744 u32 elbaf;
1745
1746 ns->pi_size = 0;
1747 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1748 if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1749 ns->pi_size = sizeof(struct t10_pi_tuple);
1750 ns->guard_type = NVME_NVM_NS_16B_GUARD;
1751 goto set_pi;
1752 }
1753
1754 nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1755 if (!nvm)
1756 return -ENOMEM;
1757
1758 c.identify.opcode = nvme_admin_identify;
1759 c.identify.nsid = cpu_to_le32(ns->head->ns_id);
1760 c.identify.cns = NVME_ID_CNS_CS_NS;
1761 c.identify.csi = NVME_CSI_NVM;
1762
1763 ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm));
1764 if (ret)
1765 goto free_data;
1766
1767 elbaf = le32_to_cpu(nvm->elbaf[lbaf]);
1768
1769 /* no support for storage tag formats right now */
1770 if (nvme_elbaf_sts(elbaf))
1771 goto free_data;
1772
1773 ns->guard_type = nvme_elbaf_guard_type(elbaf);
1774 switch (ns->guard_type) {
1775 case NVME_NVM_NS_64B_GUARD:
1776 ns->pi_size = sizeof(struct crc64_pi_tuple);
1777 break;
1778 case NVME_NVM_NS_16B_GUARD:
1779 ns->pi_size = sizeof(struct t10_pi_tuple);
1780 break;
1781 default:
1782 break;
1783 }
1784
1785 free_data:
1786 kfree(nvm);
1787 set_pi:
1788 if (ns->pi_size && (first || ns->ms == ns->pi_size))
1789 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1790 else
1791 ns->pi_type = 0;
1792
1793 return ret;
1794 }
1795
nvme_configure_metadata(struct nvme_ns * ns,struct nvme_id_ns * id)1796 static void nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1797 {
1798 struct nvme_ctrl *ctrl = ns->ctrl;
1799
1800 if (nvme_init_ms(ns, id))
1801 return;
1802
1803 ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1804 if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1805 return;
1806
1807 if (ctrl->ops->flags & NVME_F_FABRICS) {
1808 /*
1809 * The NVMe over Fabrics specification only supports metadata as
1810 * part of the extended data LBA. We rely on HCA/HBA support to
1811 * remap the separate metadata buffer from the block layer.
1812 */
1813 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1814 return;
1815
1816 ns->features |= NVME_NS_EXT_LBAS;
1817
1818 /*
1819 * The current fabrics transport drivers support namespace
1820 * metadata formats only if nvme_ns_has_pi() returns true.
1821 * Suppress support for all other formats so the namespace will
1822 * have a 0 capacity and not be usable through the block stack.
1823 *
1824 * Note, this check will need to be modified if any drivers
1825 * gain the ability to use other metadata formats.
1826 */
1827 if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns))
1828 ns->features |= NVME_NS_METADATA_SUPPORTED;
1829 } else {
1830 /*
1831 * For PCIe controllers, we can't easily remap the separate
1832 * metadata buffer from the block layer and thus require a
1833 * separate metadata buffer for block layer metadata/PI support.
1834 * We allow extended LBAs for the passthrough interface, though.
1835 */
1836 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1837 ns->features |= NVME_NS_EXT_LBAS;
1838 else
1839 ns->features |= NVME_NS_METADATA_SUPPORTED;
1840 }
1841 }
1842
nvme_set_queue_limits(struct nvme_ctrl * ctrl,struct request_queue * q)1843 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1844 struct request_queue *q)
1845 {
1846 bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1847
1848 if (ctrl->max_hw_sectors) {
1849 u32 max_segments =
1850 (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1851
1852 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1853 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1854 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1855 }
1856 blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1857 blk_queue_dma_alignment(q, 3);
1858 blk_queue_write_cache(q, vwc, vwc);
1859 }
1860
nvme_update_disk_info(struct gendisk * disk,struct nvme_ns * ns,struct nvme_id_ns * id)1861 static void nvme_update_disk_info(struct gendisk *disk,
1862 struct nvme_ns *ns, struct nvme_id_ns *id)
1863 {
1864 sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1865 unsigned short bs = 1 << ns->lba_shift;
1866 u32 atomic_bs, phys_bs, io_opt = 0;
1867
1868 /*
1869 * The block layer can't support LBA sizes larger than the page size
1870 * yet, so catch this early and don't allow block I/O.
1871 */
1872 if (ns->lba_shift > PAGE_SHIFT) {
1873 capacity = 0;
1874 bs = (1 << 9);
1875 }
1876
1877 blk_integrity_unregister(disk);
1878
1879 atomic_bs = phys_bs = bs;
1880 if (id->nabo == 0) {
1881 /*
1882 * Bit 1 indicates whether NAWUPF is defined for this namespace
1883 * and whether it should be used instead of AWUPF. If NAWUPF ==
1884 * 0 then AWUPF must be used instead.
1885 */
1886 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1887 atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1888 else
1889 atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1890 }
1891
1892 if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1893 /* NPWG = Namespace Preferred Write Granularity */
1894 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
1895 /* NOWS = Namespace Optimal Write Size */
1896 io_opt = bs * (1 + le16_to_cpu(id->nows));
1897 }
1898
1899 blk_queue_logical_block_size(disk->queue, bs);
1900 /*
1901 * Linux filesystems assume writing a single physical block is
1902 * an atomic operation. Hence limit the physical block size to the
1903 * value of the Atomic Write Unit Power Fail parameter.
1904 */
1905 blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1906 blk_queue_io_min(disk->queue, phys_bs);
1907 blk_queue_io_opt(disk->queue, io_opt);
1908
1909 /*
1910 * Register a metadata profile for PI, or the plain non-integrity NVMe
1911 * metadata masquerading as Type 0 if supported, otherwise reject block
1912 * I/O to namespaces with metadata except when the namespace supports
1913 * PI, as it can strip/insert in that case.
1914 */
1915 if (ns->ms) {
1916 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1917 (ns->features & NVME_NS_METADATA_SUPPORTED))
1918 nvme_init_integrity(disk, ns,
1919 ns->ctrl->max_integrity_segments);
1920 else if (!nvme_ns_has_pi(ns))
1921 capacity = 0;
1922 }
1923
1924 set_capacity_and_notify(disk, capacity);
1925
1926 nvme_config_discard(disk, ns);
1927 blk_queue_max_write_zeroes_sectors(disk->queue,
1928 ns->ctrl->max_zeroes_sectors);
1929 }
1930
nvme_ns_is_readonly(struct nvme_ns * ns,struct nvme_ns_info * info)1931 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info)
1932 {
1933 return info->is_readonly || test_bit(NVME_NS_FORCE_RO, &ns->flags);
1934 }
1935
nvme_first_scan(struct gendisk * disk)1936 static inline bool nvme_first_scan(struct gendisk *disk)
1937 {
1938 /* nvme_alloc_ns() scans the disk prior to adding it */
1939 return !disk_live(disk);
1940 }
1941
nvme_set_chunk_sectors(struct nvme_ns * ns,struct nvme_id_ns * id)1942 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
1943 {
1944 struct nvme_ctrl *ctrl = ns->ctrl;
1945 u32 iob;
1946
1947 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1948 is_power_of_2(ctrl->max_hw_sectors))
1949 iob = ctrl->max_hw_sectors;
1950 else
1951 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
1952
1953 if (!iob)
1954 return;
1955
1956 if (!is_power_of_2(iob)) {
1957 if (nvme_first_scan(ns->disk))
1958 pr_warn("%s: ignoring unaligned IO boundary:%u\n",
1959 ns->disk->disk_name, iob);
1960 return;
1961 }
1962
1963 if (blk_queue_is_zoned(ns->disk->queue)) {
1964 if (nvme_first_scan(ns->disk))
1965 pr_warn("%s: ignoring zoned namespace IO boundary\n",
1966 ns->disk->disk_name);
1967 return;
1968 }
1969
1970 blk_queue_chunk_sectors(ns->queue, iob);
1971 }
1972
nvme_update_ns_info_generic(struct nvme_ns * ns,struct nvme_ns_info * info)1973 static int nvme_update_ns_info_generic(struct nvme_ns *ns,
1974 struct nvme_ns_info *info)
1975 {
1976 blk_mq_freeze_queue(ns->disk->queue);
1977 nvme_set_queue_limits(ns->ctrl, ns->queue);
1978 set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
1979 blk_mq_unfreeze_queue(ns->disk->queue);
1980
1981 if (nvme_ns_head_multipath(ns->head)) {
1982 blk_mq_freeze_queue(ns->head->disk->queue);
1983 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
1984 nvme_mpath_revalidate_paths(ns);
1985 blk_stack_limits(&ns->head->disk->queue->limits,
1986 &ns->queue->limits, 0);
1987 ns->head->disk->flags |= GENHD_FL_HIDDEN;
1988 blk_mq_unfreeze_queue(ns->head->disk->queue);
1989 }
1990
1991 /* Hide the block-interface for these devices */
1992 ns->disk->flags |= GENHD_FL_HIDDEN;
1993 set_bit(NVME_NS_READY, &ns->flags);
1994
1995 return 0;
1996 }
1997
nvme_update_ns_info_block(struct nvme_ns * ns,struct nvme_ns_info * info)1998 static int nvme_update_ns_info_block(struct nvme_ns *ns,
1999 struct nvme_ns_info *info)
2000 {
2001 struct nvme_id_ns *id;
2002 unsigned lbaf;
2003 int ret;
2004
2005 ret = nvme_identify_ns(ns->ctrl, info->nsid, &id);
2006 if (ret)
2007 return ret;
2008
2009 blk_mq_freeze_queue(ns->disk->queue);
2010 lbaf = nvme_lbaf_index(id->flbas);
2011 ns->lba_shift = id->lbaf[lbaf].ds;
2012 nvme_set_queue_limits(ns->ctrl, ns->queue);
2013
2014 nvme_configure_metadata(ns, id);
2015 nvme_set_chunk_sectors(ns, id);
2016 nvme_update_disk_info(ns->disk, ns, id);
2017
2018 if (ns->head->ids.csi == NVME_CSI_ZNS) {
2019 ret = nvme_update_zone_info(ns, lbaf);
2020 if (ret) {
2021 blk_mq_unfreeze_queue(ns->disk->queue);
2022 goto out;
2023 }
2024 }
2025
2026 set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2027 set_bit(NVME_NS_READY, &ns->flags);
2028 blk_mq_unfreeze_queue(ns->disk->queue);
2029
2030 if (blk_queue_is_zoned(ns->queue)) {
2031 ret = nvme_revalidate_zones(ns);
2032 if (ret && !nvme_first_scan(ns->disk))
2033 goto out;
2034 }
2035
2036 if (nvme_ns_head_multipath(ns->head)) {
2037 blk_mq_freeze_queue(ns->head->disk->queue);
2038 nvme_update_disk_info(ns->head->disk, ns, id);
2039 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
2040 nvme_mpath_revalidate_paths(ns);
2041 blk_stack_limits(&ns->head->disk->queue->limits,
2042 &ns->queue->limits, 0);
2043 disk_update_readahead(ns->head->disk);
2044 blk_mq_unfreeze_queue(ns->head->disk->queue);
2045 }
2046
2047 ret = 0;
2048 out:
2049 /*
2050 * If probing fails due an unsupported feature, hide the block device,
2051 * but still allow other access.
2052 */
2053 if (ret == -ENODEV) {
2054 ns->disk->flags |= GENHD_FL_HIDDEN;
2055 set_bit(NVME_NS_READY, &ns->flags);
2056 ret = 0;
2057 }
2058 kfree(id);
2059 return ret;
2060 }
2061
nvme_update_ns_info(struct nvme_ns * ns,struct nvme_ns_info * info)2062 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
2063 {
2064 switch (info->ids.csi) {
2065 case NVME_CSI_ZNS:
2066 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2067 dev_info(ns->ctrl->device,
2068 "block device for nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
2069 info->nsid);
2070 return nvme_update_ns_info_generic(ns, info);
2071 }
2072 return nvme_update_ns_info_block(ns, info);
2073 case NVME_CSI_NVM:
2074 return nvme_update_ns_info_block(ns, info);
2075 default:
2076 dev_info(ns->ctrl->device,
2077 "block device for nsid %u not supported (csi %u)\n",
2078 info->nsid, info->ids.csi);
2079 return nvme_update_ns_info_generic(ns, info);
2080 }
2081 }
2082
nvme_pr_type(enum pr_type type)2083 static char nvme_pr_type(enum pr_type type)
2084 {
2085 switch (type) {
2086 case PR_WRITE_EXCLUSIVE:
2087 return 1;
2088 case PR_EXCLUSIVE_ACCESS:
2089 return 2;
2090 case PR_WRITE_EXCLUSIVE_REG_ONLY:
2091 return 3;
2092 case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2093 return 4;
2094 case PR_WRITE_EXCLUSIVE_ALL_REGS:
2095 return 5;
2096 case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2097 return 6;
2098 default:
2099 return 0;
2100 }
2101 }
2102
nvme_send_ns_head_pr_command(struct block_device * bdev,struct nvme_command * c,u8 data[16])2103 static int nvme_send_ns_head_pr_command(struct block_device *bdev,
2104 struct nvme_command *c, u8 data[16])
2105 {
2106 struct nvme_ns_head *head = bdev->bd_disk->private_data;
2107 int srcu_idx = srcu_read_lock(&head->srcu);
2108 struct nvme_ns *ns = nvme_find_path(head);
2109 int ret = -EWOULDBLOCK;
2110
2111 if (ns) {
2112 c->common.nsid = cpu_to_le32(ns->head->ns_id);
2113 ret = nvme_submit_sync_cmd(ns->queue, c, data, 16);
2114 }
2115 srcu_read_unlock(&head->srcu, srcu_idx);
2116 return ret;
2117 }
2118
nvme_send_ns_pr_command(struct nvme_ns * ns,struct nvme_command * c,u8 data[16])2119 static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c,
2120 u8 data[16])
2121 {
2122 c->common.nsid = cpu_to_le32(ns->head->ns_id);
2123 return nvme_submit_sync_cmd(ns->queue, c, data, 16);
2124 }
2125
nvme_pr_command(struct block_device * bdev,u32 cdw10,u64 key,u64 sa_key,u8 op)2126 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2127 u64 key, u64 sa_key, u8 op)
2128 {
2129 struct nvme_command c = { };
2130 u8 data[16] = { 0, };
2131
2132 put_unaligned_le64(key, &data[0]);
2133 put_unaligned_le64(sa_key, &data[8]);
2134
2135 c.common.opcode = op;
2136 c.common.cdw10 = cpu_to_le32(cdw10);
2137
2138 if (IS_ENABLED(CONFIG_NVME_MULTIPATH) &&
2139 bdev->bd_disk->fops == &nvme_ns_head_ops)
2140 return nvme_send_ns_head_pr_command(bdev, &c, data);
2141 return nvme_send_ns_pr_command(bdev->bd_disk->private_data, &c, data);
2142 }
2143
nvme_pr_register(struct block_device * bdev,u64 old,u64 new,unsigned flags)2144 static int nvme_pr_register(struct block_device *bdev, u64 old,
2145 u64 new, unsigned flags)
2146 {
2147 u32 cdw10;
2148
2149 if (flags & ~PR_FL_IGNORE_KEY)
2150 return -EOPNOTSUPP;
2151
2152 cdw10 = old ? 2 : 0;
2153 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2154 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2155 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2156 }
2157
nvme_pr_reserve(struct block_device * bdev,u64 key,enum pr_type type,unsigned flags)2158 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2159 enum pr_type type, unsigned flags)
2160 {
2161 u32 cdw10;
2162
2163 if (flags & ~PR_FL_IGNORE_KEY)
2164 return -EOPNOTSUPP;
2165
2166 cdw10 = nvme_pr_type(type) << 8;
2167 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2168 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2169 }
2170
nvme_pr_preempt(struct block_device * bdev,u64 old,u64 new,enum pr_type type,bool abort)2171 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2172 enum pr_type type, bool abort)
2173 {
2174 u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2175
2176 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2177 }
2178
nvme_pr_clear(struct block_device * bdev,u64 key)2179 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2180 {
2181 u32 cdw10 = 1 | (key ? 0 : 1 << 3);
2182
2183 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2184 }
2185
nvme_pr_release(struct block_device * bdev,u64 key,enum pr_type type)2186 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2187 {
2188 u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 0 : 1 << 3);
2189
2190 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2191 }
2192
2193 const struct pr_ops nvme_pr_ops = {
2194 .pr_register = nvme_pr_register,
2195 .pr_reserve = nvme_pr_reserve,
2196 .pr_release = nvme_pr_release,
2197 .pr_preempt = nvme_pr_preempt,
2198 .pr_clear = nvme_pr_clear,
2199 };
2200
2201 #ifdef CONFIG_BLK_SED_OPAL
nvme_sec_submit(void * data,u16 spsp,u8 secp,void * buffer,size_t len,bool send)2202 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2203 bool send)
2204 {
2205 struct nvme_ctrl *ctrl = data;
2206 struct nvme_command cmd = { };
2207
2208 if (send)
2209 cmd.common.opcode = nvme_admin_security_send;
2210 else
2211 cmd.common.opcode = nvme_admin_security_recv;
2212 cmd.common.nsid = 0;
2213 cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2214 cmd.common.cdw11 = cpu_to_le32(len);
2215
2216 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2217 NVME_QID_ANY, 1, 0);
2218 }
2219 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2220 #endif /* CONFIG_BLK_SED_OPAL */
2221
2222 #ifdef CONFIG_BLK_DEV_ZONED
nvme_report_zones(struct gendisk * disk,sector_t sector,unsigned int nr_zones,report_zones_cb cb,void * data)2223 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2224 unsigned int nr_zones, report_zones_cb cb, void *data)
2225 {
2226 return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2227 data);
2228 }
2229 #else
2230 #define nvme_report_zones NULL
2231 #endif /* CONFIG_BLK_DEV_ZONED */
2232
2233 static const struct block_device_operations nvme_bdev_ops = {
2234 .owner = THIS_MODULE,
2235 .ioctl = nvme_ioctl,
2236 .compat_ioctl = blkdev_compat_ptr_ioctl,
2237 .open = nvme_open,
2238 .release = nvme_release,
2239 .getgeo = nvme_getgeo,
2240 .report_zones = nvme_report_zones,
2241 .pr_ops = &nvme_pr_ops,
2242 };
2243
nvme_wait_ready(struct nvme_ctrl * ctrl,u32 timeout,bool enabled)2244 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 timeout, bool enabled)
2245 {
2246 unsigned long timeout_jiffies = ((timeout + 1) * HZ / 2) + jiffies;
2247 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2248 int ret;
2249
2250 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2251 if (csts == ~0)
2252 return -ENODEV;
2253 if ((csts & NVME_CSTS_RDY) == bit)
2254 break;
2255
2256 usleep_range(1000, 2000);
2257 if (fatal_signal_pending(current))
2258 return -EINTR;
2259 if (time_after(jiffies, timeout_jiffies)) {
2260 dev_err(ctrl->device,
2261 "Device not ready; aborting %s, CSTS=0x%x\n",
2262 enabled ? "initialisation" : "reset", csts);
2263 return -ENODEV;
2264 }
2265 }
2266
2267 return ret;
2268 }
2269
2270 /*
2271 * If the device has been passed off to us in an enabled state, just clear
2272 * the enabled bit. The spec says we should set the 'shutdown notification
2273 * bits', but doing so may cause the device to complete commands to the
2274 * admin queue ... and we don't know what memory that might be pointing at!
2275 */
nvme_disable_ctrl(struct nvme_ctrl * ctrl)2276 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2277 {
2278 int ret;
2279
2280 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2281 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2282
2283 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2284 if (ret)
2285 return ret;
2286
2287 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2288 msleep(NVME_QUIRK_DELAY_AMOUNT);
2289
2290 return nvme_wait_ready(ctrl, NVME_CAP_TIMEOUT(ctrl->cap), false);
2291 }
2292 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2293
nvme_enable_ctrl(struct nvme_ctrl * ctrl)2294 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2295 {
2296 unsigned dev_page_min;
2297 u32 timeout;
2298 int ret;
2299
2300 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2301 if (ret) {
2302 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2303 return ret;
2304 }
2305 dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2306
2307 if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2308 dev_err(ctrl->device,
2309 "Minimum device page size %u too large for host (%u)\n",
2310 1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2311 return -ENODEV;
2312 }
2313
2314 if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2315 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2316 else
2317 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2318
2319 if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2320 u32 crto;
2321
2322 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2323 if (ret) {
2324 dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2325 ret);
2326 return ret;
2327 }
2328
2329 if (ctrl->cap & NVME_CAP_CRMS_CRIMS) {
2330 ctrl->ctrl_config |= NVME_CC_CRIME;
2331 timeout = NVME_CRTO_CRIMT(crto);
2332 } else {
2333 timeout = NVME_CRTO_CRWMT(crto);
2334 }
2335 } else {
2336 timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2337 }
2338
2339 ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2340 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2341 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2342 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2343 if (ret)
2344 return ret;
2345
2346 /* Flush write to device (required if transport is PCI) */
2347 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CC, &ctrl->ctrl_config);
2348 if (ret)
2349 return ret;
2350
2351 ctrl->ctrl_config |= NVME_CC_ENABLE;
2352 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2353 if (ret)
2354 return ret;
2355 return nvme_wait_ready(ctrl, timeout, true);
2356 }
2357 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2358
nvme_shutdown_ctrl(struct nvme_ctrl * ctrl)2359 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2360 {
2361 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2362 u32 csts;
2363 int ret;
2364
2365 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2366 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2367
2368 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2369 if (ret)
2370 return ret;
2371
2372 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2373 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2374 break;
2375
2376 msleep(100);
2377 if (fatal_signal_pending(current))
2378 return -EINTR;
2379 if (time_after(jiffies, timeout)) {
2380 dev_err(ctrl->device,
2381 "Device shutdown incomplete; abort shutdown\n");
2382 return -ENODEV;
2383 }
2384 }
2385
2386 return ret;
2387 }
2388 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2389
nvme_configure_timestamp(struct nvme_ctrl * ctrl)2390 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2391 {
2392 __le64 ts;
2393 int ret;
2394
2395 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2396 return 0;
2397
2398 ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2399 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2400 NULL);
2401 if (ret)
2402 dev_warn_once(ctrl->device,
2403 "could not set timestamp (%d)\n", ret);
2404 return ret;
2405 }
2406
nvme_configure_host_options(struct nvme_ctrl * ctrl)2407 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2408 {
2409 struct nvme_feat_host_behavior *host;
2410 u8 acre = 0, lbafee = 0;
2411 int ret;
2412
2413 /* Don't bother enabling the feature if retry delay is not reported */
2414 if (ctrl->crdt[0])
2415 acre = NVME_ENABLE_ACRE;
2416 if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2417 lbafee = NVME_ENABLE_LBAFEE;
2418
2419 if (!acre && !lbafee)
2420 return 0;
2421
2422 host = kzalloc(sizeof(*host), GFP_KERNEL);
2423 if (!host)
2424 return 0;
2425
2426 host->acre = acre;
2427 host->lbafee = lbafee;
2428 ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2429 host, sizeof(*host), NULL);
2430 kfree(host);
2431 return ret;
2432 }
2433
2434 /*
2435 * The function checks whether the given total (exlat + enlat) latency of
2436 * a power state allows the latter to be used as an APST transition target.
2437 * It does so by comparing the latency to the primary and secondary latency
2438 * tolerances defined by module params. If there's a match, the corresponding
2439 * timeout value is returned and the matching tolerance index (1 or 2) is
2440 * reported.
2441 */
nvme_apst_get_transition_time(u64 total_latency,u64 * transition_time,unsigned * last_index)2442 static bool nvme_apst_get_transition_time(u64 total_latency,
2443 u64 *transition_time, unsigned *last_index)
2444 {
2445 if (total_latency <= apst_primary_latency_tol_us) {
2446 if (*last_index == 1)
2447 return false;
2448 *last_index = 1;
2449 *transition_time = apst_primary_timeout_ms;
2450 return true;
2451 }
2452 if (apst_secondary_timeout_ms &&
2453 total_latency <= apst_secondary_latency_tol_us) {
2454 if (*last_index <= 2)
2455 return false;
2456 *last_index = 2;
2457 *transition_time = apst_secondary_timeout_ms;
2458 return true;
2459 }
2460 return false;
2461 }
2462
2463 /*
2464 * APST (Autonomous Power State Transition) lets us program a table of power
2465 * state transitions that the controller will perform automatically.
2466 *
2467 * Depending on module params, one of the two supported techniques will be used:
2468 *
2469 * - If the parameters provide explicit timeouts and tolerances, they will be
2470 * used to build a table with up to 2 non-operational states to transition to.
2471 * The default parameter values were selected based on the values used by
2472 * Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2473 * regeneration of the APST table in the event of switching between external
2474 * and battery power, the timeouts and tolerances reflect a compromise
2475 * between values used by Microsoft for AC and battery scenarios.
2476 * - If not, we'll configure the table with a simple heuristic: we are willing
2477 * to spend at most 2% of the time transitioning between power states.
2478 * Therefore, when running in any given state, we will enter the next
2479 * lower-power non-operational state after waiting 50 * (enlat + exlat)
2480 * microseconds, as long as that state's exit latency is under the requested
2481 * maximum latency.
2482 *
2483 * We will not autonomously enter any non-operational state for which the total
2484 * latency exceeds ps_max_latency_us.
2485 *
2486 * Users can set ps_max_latency_us to zero to turn off APST.
2487 */
nvme_configure_apst(struct nvme_ctrl * ctrl)2488 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2489 {
2490 struct nvme_feat_auto_pst *table;
2491 unsigned apste = 0;
2492 u64 max_lat_us = 0;
2493 __le64 target = 0;
2494 int max_ps = -1;
2495 int state;
2496 int ret;
2497 unsigned last_lt_index = UINT_MAX;
2498
2499 /*
2500 * If APST isn't supported or if we haven't been initialized yet,
2501 * then don't do anything.
2502 */
2503 if (!ctrl->apsta)
2504 return 0;
2505
2506 if (ctrl->npss > 31) {
2507 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2508 return 0;
2509 }
2510
2511 table = kzalloc(sizeof(*table), GFP_KERNEL);
2512 if (!table)
2513 return 0;
2514
2515 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2516 /* Turn off APST. */
2517 dev_dbg(ctrl->device, "APST disabled\n");
2518 goto done;
2519 }
2520
2521 /*
2522 * Walk through all states from lowest- to highest-power.
2523 * According to the spec, lower-numbered states use more power. NPSS,
2524 * despite the name, is the index of the lowest-power state, not the
2525 * number of states.
2526 */
2527 for (state = (int)ctrl->npss; state >= 0; state--) {
2528 u64 total_latency_us, exit_latency_us, transition_ms;
2529
2530 if (target)
2531 table->entries[state] = target;
2532
2533 /*
2534 * Don't allow transitions to the deepest state if it's quirked
2535 * off.
2536 */
2537 if (state == ctrl->npss &&
2538 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2539 continue;
2540
2541 /*
2542 * Is this state a useful non-operational state for higher-power
2543 * states to autonomously transition to?
2544 */
2545 if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2546 continue;
2547
2548 exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2549 if (exit_latency_us > ctrl->ps_max_latency_us)
2550 continue;
2551
2552 total_latency_us = exit_latency_us +
2553 le32_to_cpu(ctrl->psd[state].entry_lat);
2554
2555 /*
2556 * This state is good. It can be used as the APST idle target
2557 * for higher power states.
2558 */
2559 if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2560 if (!nvme_apst_get_transition_time(total_latency_us,
2561 &transition_ms, &last_lt_index))
2562 continue;
2563 } else {
2564 transition_ms = total_latency_us + 19;
2565 do_div(transition_ms, 20);
2566 if (transition_ms > (1 << 24) - 1)
2567 transition_ms = (1 << 24) - 1;
2568 }
2569
2570 target = cpu_to_le64((state << 3) | (transition_ms << 8));
2571 if (max_ps == -1)
2572 max_ps = state;
2573 if (total_latency_us > max_lat_us)
2574 max_lat_us = total_latency_us;
2575 }
2576
2577 if (max_ps == -1)
2578 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2579 else
2580 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2581 max_ps, max_lat_us, (int)sizeof(*table), table);
2582 apste = 1;
2583
2584 done:
2585 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2586 table, sizeof(*table), NULL);
2587 if (ret)
2588 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2589 kfree(table);
2590 return ret;
2591 }
2592
nvme_set_latency_tolerance(struct device * dev,s32 val)2593 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2594 {
2595 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2596 u64 latency;
2597
2598 switch (val) {
2599 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2600 case PM_QOS_LATENCY_ANY:
2601 latency = U64_MAX;
2602 break;
2603
2604 default:
2605 latency = val;
2606 }
2607
2608 if (ctrl->ps_max_latency_us != latency) {
2609 ctrl->ps_max_latency_us = latency;
2610 if (ctrl->state == NVME_CTRL_LIVE)
2611 nvme_configure_apst(ctrl);
2612 }
2613 }
2614
2615 struct nvme_core_quirk_entry {
2616 /*
2617 * NVMe model and firmware strings are padded with spaces. For
2618 * simplicity, strings in the quirk table are padded with NULLs
2619 * instead.
2620 */
2621 u16 vid;
2622 const char *mn;
2623 const char *fr;
2624 unsigned long quirks;
2625 };
2626
2627 static const struct nvme_core_quirk_entry core_quirks[] = {
2628 {
2629 /*
2630 * This Toshiba device seems to die using any APST states. See:
2631 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2632 */
2633 .vid = 0x1179,
2634 .mn = "THNSF5256GPUK TOSHIBA",
2635 .quirks = NVME_QUIRK_NO_APST,
2636 },
2637 {
2638 /*
2639 * This LiteON CL1-3D*-Q11 firmware version has a race
2640 * condition associated with actions related to suspend to idle
2641 * LiteON has resolved the problem in future firmware
2642 */
2643 .vid = 0x14a4,
2644 .fr = "22301111",
2645 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2646 },
2647 {
2648 /*
2649 * This Kioxia CD6-V Series / HPE PE8030 device times out and
2650 * aborts I/O during any load, but more easily reproducible
2651 * with discards (fstrim).
2652 *
2653 * The device is left in a state where it is also not possible
2654 * to use "nvme set-feature" to disable APST, but booting with
2655 * nvme_core.default_ps_max_latency=0 works.
2656 */
2657 .vid = 0x1e0f,
2658 .mn = "KCD6XVUL6T40",
2659 .quirks = NVME_QUIRK_NO_APST,
2660 },
2661 {
2662 /*
2663 * The external Samsung X5 SSD fails initialization without a
2664 * delay before checking if it is ready and has a whole set of
2665 * other problems. To make this even more interesting, it
2666 * shares the PCI ID with internal Samsung 970 Evo Plus that
2667 * does not need or want these quirks.
2668 */
2669 .vid = 0x144d,
2670 .mn = "Samsung Portable SSD X5",
2671 .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2672 NVME_QUIRK_NO_DEEPEST_PS |
2673 NVME_QUIRK_IGNORE_DEV_SUBNQN,
2674 }
2675 };
2676
2677 /* match is null-terminated but idstr is space-padded. */
string_matches(const char * idstr,const char * match,size_t len)2678 static bool string_matches(const char *idstr, const char *match, size_t len)
2679 {
2680 size_t matchlen;
2681
2682 if (!match)
2683 return true;
2684
2685 matchlen = strlen(match);
2686 WARN_ON_ONCE(matchlen > len);
2687
2688 if (memcmp(idstr, match, matchlen))
2689 return false;
2690
2691 for (; matchlen < len; matchlen++)
2692 if (idstr[matchlen] != ' ')
2693 return false;
2694
2695 return true;
2696 }
2697
quirk_matches(const struct nvme_id_ctrl * id,const struct nvme_core_quirk_entry * q)2698 static bool quirk_matches(const struct nvme_id_ctrl *id,
2699 const struct nvme_core_quirk_entry *q)
2700 {
2701 return q->vid == le16_to_cpu(id->vid) &&
2702 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2703 string_matches(id->fr, q->fr, sizeof(id->fr));
2704 }
2705
nvme_init_subnqn(struct nvme_subsystem * subsys,struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)2706 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2707 struct nvme_id_ctrl *id)
2708 {
2709 size_t nqnlen;
2710 int off;
2711
2712 if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2713 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2714 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2715 strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2716 return;
2717 }
2718
2719 if (ctrl->vs >= NVME_VS(1, 2, 1))
2720 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2721 }
2722
2723 /*
2724 * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe
2725 * Base Specification 2.0. It is slightly different from the format
2726 * specified there due to historic reasons, and we can't change it now.
2727 */
2728 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2729 "nqn.2014.08.org.nvmexpress:%04x%04x",
2730 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2731 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2732 off += sizeof(id->sn);
2733 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2734 off += sizeof(id->mn);
2735 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2736 }
2737
nvme_release_subsystem(struct device * dev)2738 static void nvme_release_subsystem(struct device *dev)
2739 {
2740 struct nvme_subsystem *subsys =
2741 container_of(dev, struct nvme_subsystem, dev);
2742
2743 if (subsys->instance >= 0)
2744 ida_free(&nvme_instance_ida, subsys->instance);
2745 kfree(subsys);
2746 }
2747
nvme_destroy_subsystem(struct kref * ref)2748 static void nvme_destroy_subsystem(struct kref *ref)
2749 {
2750 struct nvme_subsystem *subsys =
2751 container_of(ref, struct nvme_subsystem, ref);
2752
2753 mutex_lock(&nvme_subsystems_lock);
2754 list_del(&subsys->entry);
2755 mutex_unlock(&nvme_subsystems_lock);
2756
2757 ida_destroy(&subsys->ns_ida);
2758 device_del(&subsys->dev);
2759 put_device(&subsys->dev);
2760 }
2761
nvme_put_subsystem(struct nvme_subsystem * subsys)2762 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2763 {
2764 kref_put(&subsys->ref, nvme_destroy_subsystem);
2765 }
2766
__nvme_find_get_subsystem(const char * subsysnqn)2767 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2768 {
2769 struct nvme_subsystem *subsys;
2770
2771 lockdep_assert_held(&nvme_subsystems_lock);
2772
2773 /*
2774 * Fail matches for discovery subsystems. This results
2775 * in each discovery controller bound to a unique subsystem.
2776 * This avoids issues with validating controller values
2777 * that can only be true when there is a single unique subsystem.
2778 * There may be multiple and completely independent entities
2779 * that provide discovery controllers.
2780 */
2781 if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2782 return NULL;
2783
2784 list_for_each_entry(subsys, &nvme_subsystems, entry) {
2785 if (strcmp(subsys->subnqn, subsysnqn))
2786 continue;
2787 if (!kref_get_unless_zero(&subsys->ref))
2788 continue;
2789 return subsys;
2790 }
2791
2792 return NULL;
2793 }
2794
2795 #define SUBSYS_ATTR_RO(_name, _mode, _show) \
2796 struct device_attribute subsys_attr_##_name = \
2797 __ATTR(_name, _mode, _show, NULL)
2798
nvme_subsys_show_nqn(struct device * dev,struct device_attribute * attr,char * buf)2799 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2800 struct device_attribute *attr,
2801 char *buf)
2802 {
2803 struct nvme_subsystem *subsys =
2804 container_of(dev, struct nvme_subsystem, dev);
2805
2806 return sysfs_emit(buf, "%s\n", subsys->subnqn);
2807 }
2808 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2809
nvme_subsys_show_type(struct device * dev,struct device_attribute * attr,char * buf)2810 static ssize_t nvme_subsys_show_type(struct device *dev,
2811 struct device_attribute *attr,
2812 char *buf)
2813 {
2814 struct nvme_subsystem *subsys =
2815 container_of(dev, struct nvme_subsystem, dev);
2816
2817 switch (subsys->subtype) {
2818 case NVME_NQN_DISC:
2819 return sysfs_emit(buf, "discovery\n");
2820 case NVME_NQN_NVME:
2821 return sysfs_emit(buf, "nvm\n");
2822 default:
2823 return sysfs_emit(buf, "reserved\n");
2824 }
2825 }
2826 static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type);
2827
2828 #define nvme_subsys_show_str_function(field) \
2829 static ssize_t subsys_##field##_show(struct device *dev, \
2830 struct device_attribute *attr, char *buf) \
2831 { \
2832 struct nvme_subsystem *subsys = \
2833 container_of(dev, struct nvme_subsystem, dev); \
2834 return sysfs_emit(buf, "%.*s\n", \
2835 (int)sizeof(subsys->field), subsys->field); \
2836 } \
2837 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2838
2839 nvme_subsys_show_str_function(model);
2840 nvme_subsys_show_str_function(serial);
2841 nvme_subsys_show_str_function(firmware_rev);
2842
2843 static struct attribute *nvme_subsys_attrs[] = {
2844 &subsys_attr_model.attr,
2845 &subsys_attr_serial.attr,
2846 &subsys_attr_firmware_rev.attr,
2847 &subsys_attr_subsysnqn.attr,
2848 &subsys_attr_subsystype.attr,
2849 #ifdef CONFIG_NVME_MULTIPATH
2850 &subsys_attr_iopolicy.attr,
2851 #endif
2852 NULL,
2853 };
2854
2855 static const struct attribute_group nvme_subsys_attrs_group = {
2856 .attrs = nvme_subsys_attrs,
2857 };
2858
2859 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2860 &nvme_subsys_attrs_group,
2861 NULL,
2862 };
2863
nvme_discovery_ctrl(struct nvme_ctrl * ctrl)2864 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2865 {
2866 return ctrl->opts && ctrl->opts->discovery_nqn;
2867 }
2868
nvme_validate_cntlid(struct nvme_subsystem * subsys,struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)2869 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2870 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2871 {
2872 struct nvme_ctrl *tmp;
2873
2874 lockdep_assert_held(&nvme_subsystems_lock);
2875
2876 list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2877 if (nvme_state_terminal(tmp))
2878 continue;
2879
2880 if (tmp->cntlid == ctrl->cntlid) {
2881 dev_err(ctrl->device,
2882 "Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2883 ctrl->cntlid, dev_name(tmp->device),
2884 subsys->subnqn);
2885 return false;
2886 }
2887
2888 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2889 nvme_discovery_ctrl(ctrl))
2890 continue;
2891
2892 dev_err(ctrl->device,
2893 "Subsystem does not support multiple controllers\n");
2894 return false;
2895 }
2896
2897 return true;
2898 }
2899
nvme_init_subsystem(struct nvme_ctrl * ctrl,struct nvme_id_ctrl * id)2900 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2901 {
2902 struct nvme_subsystem *subsys, *found;
2903 int ret;
2904
2905 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2906 if (!subsys)
2907 return -ENOMEM;
2908
2909 subsys->instance = -1;
2910 mutex_init(&subsys->lock);
2911 kref_init(&subsys->ref);
2912 INIT_LIST_HEAD(&subsys->ctrls);
2913 INIT_LIST_HEAD(&subsys->nsheads);
2914 nvme_init_subnqn(subsys, ctrl, id);
2915 memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2916 memcpy(subsys->model, id->mn, sizeof(subsys->model));
2917 subsys->vendor_id = le16_to_cpu(id->vid);
2918 subsys->cmic = id->cmic;
2919
2920 /* Versions prior to 1.4 don't necessarily report a valid type */
2921 if (id->cntrltype == NVME_CTRL_DISC ||
2922 !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2923 subsys->subtype = NVME_NQN_DISC;
2924 else
2925 subsys->subtype = NVME_NQN_NVME;
2926
2927 if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2928 dev_err(ctrl->device,
2929 "Subsystem %s is not a discovery controller",
2930 subsys->subnqn);
2931 kfree(subsys);
2932 return -EINVAL;
2933 }
2934 subsys->awupf = le16_to_cpu(id->awupf);
2935 nvme_mpath_default_iopolicy(subsys);
2936
2937 subsys->dev.class = nvme_subsys_class;
2938 subsys->dev.release = nvme_release_subsystem;
2939 subsys->dev.groups = nvme_subsys_attrs_groups;
2940 dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2941 device_initialize(&subsys->dev);
2942
2943 mutex_lock(&nvme_subsystems_lock);
2944 found = __nvme_find_get_subsystem(subsys->subnqn);
2945 if (found) {
2946 put_device(&subsys->dev);
2947 subsys = found;
2948
2949 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2950 ret = -EINVAL;
2951 goto out_put_subsystem;
2952 }
2953 } else {
2954 ret = device_add(&subsys->dev);
2955 if (ret) {
2956 dev_err(ctrl->device,
2957 "failed to register subsystem device.\n");
2958 put_device(&subsys->dev);
2959 goto out_unlock;
2960 }
2961 ida_init(&subsys->ns_ida);
2962 list_add_tail(&subsys->entry, &nvme_subsystems);
2963 }
2964
2965 ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2966 dev_name(ctrl->device));
2967 if (ret) {
2968 dev_err(ctrl->device,
2969 "failed to create sysfs link from subsystem.\n");
2970 goto out_put_subsystem;
2971 }
2972
2973 if (!found)
2974 subsys->instance = ctrl->instance;
2975 ctrl->subsys = subsys;
2976 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2977 mutex_unlock(&nvme_subsystems_lock);
2978 return 0;
2979
2980 out_put_subsystem:
2981 nvme_put_subsystem(subsys);
2982 out_unlock:
2983 mutex_unlock(&nvme_subsystems_lock);
2984 return ret;
2985 }
2986
nvme_get_log(struct nvme_ctrl * ctrl,u32 nsid,u8 log_page,u8 lsp,u8 csi,void * log,size_t size,u64 offset)2987 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2988 void *log, size_t size, u64 offset)
2989 {
2990 struct nvme_command c = { };
2991 u32 dwlen = nvme_bytes_to_numd(size);
2992
2993 c.get_log_page.opcode = nvme_admin_get_log_page;
2994 c.get_log_page.nsid = cpu_to_le32(nsid);
2995 c.get_log_page.lid = log_page;
2996 c.get_log_page.lsp = lsp;
2997 c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2998 c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2999 c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
3000 c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3001 c.get_log_page.csi = csi;
3002
3003 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3004 }
3005
nvme_get_effects_log(struct nvme_ctrl * ctrl,u8 csi,struct nvme_effects_log ** log)3006 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3007 struct nvme_effects_log **log)
3008 {
3009 struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
3010 int ret;
3011
3012 if (cel)
3013 goto out;
3014
3015 cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3016 if (!cel)
3017 return -ENOMEM;
3018
3019 ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
3020 cel, sizeof(*cel), 0);
3021 if (ret) {
3022 kfree(cel);
3023 return ret;
3024 }
3025
3026 xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
3027 out:
3028 *log = cel;
3029 return 0;
3030 }
3031
nvme_mps_to_sectors(struct nvme_ctrl * ctrl,u32 units)3032 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
3033 {
3034 u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
3035
3036 if (check_shl_overflow(1U, units + page_shift - 9, &val))
3037 return UINT_MAX;
3038 return val;
3039 }
3040
nvme_init_non_mdts_limits(struct nvme_ctrl * ctrl)3041 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
3042 {
3043 struct nvme_command c = { };
3044 struct nvme_id_ctrl_nvm *id;
3045 int ret;
3046
3047 if (ctrl->oncs & NVME_CTRL_ONCS_DSM) {
3048 ctrl->max_discard_sectors = UINT_MAX;
3049 ctrl->max_discard_segments = NVME_DSM_MAX_RANGES;
3050 } else {
3051 ctrl->max_discard_sectors = 0;
3052 ctrl->max_discard_segments = 0;
3053 }
3054
3055 /*
3056 * Even though NVMe spec explicitly states that MDTS is not applicable
3057 * to the write-zeroes, we are cautious and limit the size to the
3058 * controllers max_hw_sectors value, which is based on the MDTS field
3059 * and possibly other limiting factors.
3060 */
3061 if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
3062 !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
3063 ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
3064 else
3065 ctrl->max_zeroes_sectors = 0;
3066
3067 if (nvme_ctrl_limited_cns(ctrl))
3068 return 0;
3069
3070 id = kzalloc(sizeof(*id), GFP_KERNEL);
3071 if (!id)
3072 return -ENOMEM;
3073
3074 c.identify.opcode = nvme_admin_identify;
3075 c.identify.cns = NVME_ID_CNS_CS_CTRL;
3076 c.identify.csi = NVME_CSI_NVM;
3077
3078 ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
3079 if (ret)
3080 goto free_data;
3081
3082 if (id->dmrl)
3083 ctrl->max_discard_segments = id->dmrl;
3084 ctrl->dmrsl = le32_to_cpu(id->dmrsl);
3085 if (id->wzsl)
3086 ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
3087
3088 free_data:
3089 kfree(id);
3090 return ret;
3091 }
3092
nvme_init_identify(struct nvme_ctrl * ctrl)3093 static int nvme_init_identify(struct nvme_ctrl *ctrl)
3094 {
3095 struct nvme_id_ctrl *id;
3096 u32 max_hw_sectors;
3097 bool prev_apst_enabled;
3098 int ret;
3099
3100 ret = nvme_identify_ctrl(ctrl, &id);
3101 if (ret) {
3102 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3103 return -EIO;
3104 }
3105
3106 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3107 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3108 if (ret < 0)
3109 goto out_free;
3110 }
3111
3112 if (!(ctrl->ops->flags & NVME_F_FABRICS))
3113 ctrl->cntlid = le16_to_cpu(id->cntlid);
3114
3115 if (!ctrl->identified) {
3116 unsigned int i;
3117
3118 /*
3119 * Check for quirks. Quirk can depend on firmware version,
3120 * so, in principle, the set of quirks present can change
3121 * across a reset. As a possible future enhancement, we
3122 * could re-scan for quirks every time we reinitialize
3123 * the device, but we'd have to make sure that the driver
3124 * behaves intelligently if the quirks change.
3125 */
3126 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3127 if (quirk_matches(id, &core_quirks[i]))
3128 ctrl->quirks |= core_quirks[i].quirks;
3129 }
3130
3131 ret = nvme_init_subsystem(ctrl, id);
3132 if (ret)
3133 goto out_free;
3134 }
3135 memcpy(ctrl->subsys->firmware_rev, id->fr,
3136 sizeof(ctrl->subsys->firmware_rev));
3137
3138 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3139 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3140 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3141 }
3142
3143 ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3144 ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3145 ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3146
3147 ctrl->oacs = le16_to_cpu(id->oacs);
3148 ctrl->oncs = le16_to_cpu(id->oncs);
3149 ctrl->mtfa = le16_to_cpu(id->mtfa);
3150 ctrl->oaes = le32_to_cpu(id->oaes);
3151 ctrl->wctemp = le16_to_cpu(id->wctemp);
3152 ctrl->cctemp = le16_to_cpu(id->cctemp);
3153
3154 atomic_set(&ctrl->abort_limit, id->acl + 1);
3155 ctrl->vwc = id->vwc;
3156 if (id->mdts)
3157 max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3158 else
3159 max_hw_sectors = UINT_MAX;
3160 ctrl->max_hw_sectors =
3161 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3162
3163 nvme_set_queue_limits(ctrl, ctrl->admin_q);
3164 ctrl->sgls = le32_to_cpu(id->sgls);
3165 ctrl->kas = le16_to_cpu(id->kas);
3166 ctrl->max_namespaces = le32_to_cpu(id->mnan);
3167 ctrl->ctratt = le32_to_cpu(id->ctratt);
3168
3169 ctrl->cntrltype = id->cntrltype;
3170 ctrl->dctype = id->dctype;
3171
3172 if (id->rtd3e) {
3173 /* us -> s */
3174 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3175
3176 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3177 shutdown_timeout, 60);
3178
3179 if (ctrl->shutdown_timeout != shutdown_timeout)
3180 dev_info(ctrl->device,
3181 "Shutdown timeout set to %u seconds\n",
3182 ctrl->shutdown_timeout);
3183 } else
3184 ctrl->shutdown_timeout = shutdown_timeout;
3185
3186 ctrl->npss = id->npss;
3187 ctrl->apsta = id->apsta;
3188 prev_apst_enabled = ctrl->apst_enabled;
3189 if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3190 if (force_apst && id->apsta) {
3191 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3192 ctrl->apst_enabled = true;
3193 } else {
3194 ctrl->apst_enabled = false;
3195 }
3196 } else {
3197 ctrl->apst_enabled = id->apsta;
3198 }
3199 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3200
3201 if (ctrl->ops->flags & NVME_F_FABRICS) {
3202 ctrl->icdoff = le16_to_cpu(id->icdoff);
3203 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3204 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3205 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3206
3207 /*
3208 * In fabrics we need to verify the cntlid matches the
3209 * admin connect
3210 */
3211 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3212 dev_err(ctrl->device,
3213 "Mismatching cntlid: Connect %u vs Identify "
3214 "%u, rejecting\n",
3215 ctrl->cntlid, le16_to_cpu(id->cntlid));
3216 ret = -EINVAL;
3217 goto out_free;
3218 }
3219
3220 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3221 dev_err(ctrl->device,
3222 "keep-alive support is mandatory for fabrics\n");
3223 ret = -EINVAL;
3224 goto out_free;
3225 }
3226 } else {
3227 ctrl->hmpre = le32_to_cpu(id->hmpre);
3228 ctrl->hmmin = le32_to_cpu(id->hmmin);
3229 ctrl->hmminds = le32_to_cpu(id->hmminds);
3230 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3231 }
3232
3233 ret = nvme_mpath_init_identify(ctrl, id);
3234 if (ret < 0)
3235 goto out_free;
3236
3237 if (ctrl->apst_enabled && !prev_apst_enabled)
3238 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3239 else if (!ctrl->apst_enabled && prev_apst_enabled)
3240 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3241
3242 out_free:
3243 kfree(id);
3244 return ret;
3245 }
3246
3247 /*
3248 * Initialize the cached copies of the Identify data and various controller
3249 * register in our nvme_ctrl structure. This should be called as soon as
3250 * the admin queue is fully up and running.
3251 */
nvme_init_ctrl_finish(struct nvme_ctrl * ctrl)3252 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl)
3253 {
3254 int ret;
3255
3256 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3257 if (ret) {
3258 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3259 return ret;
3260 }
3261
3262 ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3263
3264 if (ctrl->vs >= NVME_VS(1, 1, 0))
3265 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3266
3267 ret = nvme_init_identify(ctrl);
3268 if (ret)
3269 return ret;
3270
3271 ret = nvme_configure_apst(ctrl);
3272 if (ret < 0)
3273 return ret;
3274
3275 ret = nvme_configure_timestamp(ctrl);
3276 if (ret < 0)
3277 return ret;
3278
3279 ret = nvme_configure_host_options(ctrl);
3280 if (ret < 0)
3281 return ret;
3282
3283 if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3284 /*
3285 * Do not return errors unless we are in a controller reset,
3286 * the controller works perfectly fine without hwmon.
3287 */
3288 ret = nvme_hwmon_init(ctrl);
3289 if (ret == -EINTR)
3290 return ret;
3291 }
3292
3293 ctrl->identified = true;
3294
3295 return 0;
3296 }
3297 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3298
nvme_dev_open(struct inode * inode,struct file * file)3299 static int nvme_dev_open(struct inode *inode, struct file *file)
3300 {
3301 struct nvme_ctrl *ctrl =
3302 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3303
3304 switch (ctrl->state) {
3305 case NVME_CTRL_LIVE:
3306 break;
3307 default:
3308 return -EWOULDBLOCK;
3309 }
3310
3311 nvme_get_ctrl(ctrl);
3312 if (!try_module_get(ctrl->ops->module)) {
3313 nvme_put_ctrl(ctrl);
3314 return -EINVAL;
3315 }
3316
3317 file->private_data = ctrl;
3318 return 0;
3319 }
3320
nvme_dev_release(struct inode * inode,struct file * file)3321 static int nvme_dev_release(struct inode *inode, struct file *file)
3322 {
3323 struct nvme_ctrl *ctrl =
3324 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3325
3326 module_put(ctrl->ops->module);
3327 nvme_put_ctrl(ctrl);
3328 return 0;
3329 }
3330
3331 static const struct file_operations nvme_dev_fops = {
3332 .owner = THIS_MODULE,
3333 .open = nvme_dev_open,
3334 .release = nvme_dev_release,
3335 .unlocked_ioctl = nvme_dev_ioctl,
3336 .compat_ioctl = compat_ptr_ioctl,
3337 .uring_cmd = nvme_dev_uring_cmd,
3338 };
3339
nvme_sysfs_reset(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3340 static ssize_t nvme_sysfs_reset(struct device *dev,
3341 struct device_attribute *attr, const char *buf,
3342 size_t count)
3343 {
3344 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3345 int ret;
3346
3347 ret = nvme_reset_ctrl_sync(ctrl);
3348 if (ret < 0)
3349 return ret;
3350 return count;
3351 }
3352 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3353
nvme_sysfs_rescan(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3354 static ssize_t nvme_sysfs_rescan(struct device *dev,
3355 struct device_attribute *attr, const char *buf,
3356 size_t count)
3357 {
3358 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3359
3360 nvme_queue_scan(ctrl);
3361 return count;
3362 }
3363 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3364
dev_to_ns_head(struct device * dev)3365 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3366 {
3367 struct gendisk *disk = dev_to_disk(dev);
3368
3369 if (disk->fops == &nvme_bdev_ops)
3370 return nvme_get_ns_from_dev(dev)->head;
3371 else
3372 return disk->private_data;
3373 }
3374
wwid_show(struct device * dev,struct device_attribute * attr,char * buf)3375 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3376 char *buf)
3377 {
3378 struct nvme_ns_head *head = dev_to_ns_head(dev);
3379 struct nvme_ns_ids *ids = &head->ids;
3380 struct nvme_subsystem *subsys = head->subsys;
3381 int serial_len = sizeof(subsys->serial);
3382 int model_len = sizeof(subsys->model);
3383
3384 if (!uuid_is_null(&ids->uuid))
3385 return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
3386
3387 if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3388 return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
3389
3390 if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3391 return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
3392
3393 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3394 subsys->serial[serial_len - 1] == '\0'))
3395 serial_len--;
3396 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3397 subsys->model[model_len - 1] == '\0'))
3398 model_len--;
3399
3400 return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3401 serial_len, subsys->serial, model_len, subsys->model,
3402 head->ns_id);
3403 }
3404 static DEVICE_ATTR_RO(wwid);
3405
nguid_show(struct device * dev,struct device_attribute * attr,char * buf)3406 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3407 char *buf)
3408 {
3409 return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3410 }
3411 static DEVICE_ATTR_RO(nguid);
3412
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)3413 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3414 char *buf)
3415 {
3416 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3417
3418 /* For backward compatibility expose the NGUID to userspace if
3419 * we have no UUID set
3420 */
3421 if (uuid_is_null(&ids->uuid)) {
3422 dev_warn_ratelimited(dev,
3423 "No UUID available providing old NGUID\n");
3424 return sysfs_emit(buf, "%pU\n", ids->nguid);
3425 }
3426 return sysfs_emit(buf, "%pU\n", &ids->uuid);
3427 }
3428 static DEVICE_ATTR_RO(uuid);
3429
eui_show(struct device * dev,struct device_attribute * attr,char * buf)3430 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3431 char *buf)
3432 {
3433 return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3434 }
3435 static DEVICE_ATTR_RO(eui);
3436
nsid_show(struct device * dev,struct device_attribute * attr,char * buf)3437 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3438 char *buf)
3439 {
3440 return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3441 }
3442 static DEVICE_ATTR_RO(nsid);
3443
3444 static struct attribute *nvme_ns_id_attrs[] = {
3445 &dev_attr_wwid.attr,
3446 &dev_attr_uuid.attr,
3447 &dev_attr_nguid.attr,
3448 &dev_attr_eui.attr,
3449 &dev_attr_nsid.attr,
3450 #ifdef CONFIG_NVME_MULTIPATH
3451 &dev_attr_ana_grpid.attr,
3452 &dev_attr_ana_state.attr,
3453 #endif
3454 NULL,
3455 };
3456
nvme_ns_id_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)3457 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3458 struct attribute *a, int n)
3459 {
3460 struct device *dev = container_of(kobj, struct device, kobj);
3461 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3462
3463 if (a == &dev_attr_uuid.attr) {
3464 if (uuid_is_null(&ids->uuid) &&
3465 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3466 return 0;
3467 }
3468 if (a == &dev_attr_nguid.attr) {
3469 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3470 return 0;
3471 }
3472 if (a == &dev_attr_eui.attr) {
3473 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3474 return 0;
3475 }
3476 #ifdef CONFIG_NVME_MULTIPATH
3477 if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3478 if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */
3479 return 0;
3480 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3481 return 0;
3482 }
3483 #endif
3484 return a->mode;
3485 }
3486
3487 static const struct attribute_group nvme_ns_id_attr_group = {
3488 .attrs = nvme_ns_id_attrs,
3489 .is_visible = nvme_ns_id_attrs_are_visible,
3490 };
3491
3492 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3493 &nvme_ns_id_attr_group,
3494 NULL,
3495 };
3496
3497 #define nvme_show_str_function(field) \
3498 static ssize_t field##_show(struct device *dev, \
3499 struct device_attribute *attr, char *buf) \
3500 { \
3501 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
3502 return sysfs_emit(buf, "%.*s\n", \
3503 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
3504 } \
3505 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3506
3507 nvme_show_str_function(model);
3508 nvme_show_str_function(serial);
3509 nvme_show_str_function(firmware_rev);
3510
3511 #define nvme_show_int_function(field) \
3512 static ssize_t field##_show(struct device *dev, \
3513 struct device_attribute *attr, char *buf) \
3514 { \
3515 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
3516 return sysfs_emit(buf, "%d\n", ctrl->field); \
3517 } \
3518 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3519
3520 nvme_show_int_function(cntlid);
3521 nvme_show_int_function(numa_node);
3522 nvme_show_int_function(queue_count);
3523 nvme_show_int_function(sqsize);
3524 nvme_show_int_function(kato);
3525
nvme_sysfs_delete(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3526 static ssize_t nvme_sysfs_delete(struct device *dev,
3527 struct device_attribute *attr, const char *buf,
3528 size_t count)
3529 {
3530 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3531
3532 if (device_remove_file_self(dev, attr))
3533 nvme_delete_ctrl_sync(ctrl);
3534 return count;
3535 }
3536 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3537
nvme_sysfs_show_transport(struct device * dev,struct device_attribute * attr,char * buf)3538 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3539 struct device_attribute *attr,
3540 char *buf)
3541 {
3542 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3543
3544 return sysfs_emit(buf, "%s\n", ctrl->ops->name);
3545 }
3546 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3547
nvme_sysfs_show_state(struct device * dev,struct device_attribute * attr,char * buf)3548 static ssize_t nvme_sysfs_show_state(struct device *dev,
3549 struct device_attribute *attr,
3550 char *buf)
3551 {
3552 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3553 static const char *const state_name[] = {
3554 [NVME_CTRL_NEW] = "new",
3555 [NVME_CTRL_LIVE] = "live",
3556 [NVME_CTRL_RESETTING] = "resetting",
3557 [NVME_CTRL_CONNECTING] = "connecting",
3558 [NVME_CTRL_DELETING] = "deleting",
3559 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3560 [NVME_CTRL_DEAD] = "dead",
3561 };
3562
3563 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3564 state_name[ctrl->state])
3565 return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
3566
3567 return sysfs_emit(buf, "unknown state\n");
3568 }
3569
3570 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3571
nvme_sysfs_show_subsysnqn(struct device * dev,struct device_attribute * attr,char * buf)3572 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3573 struct device_attribute *attr,
3574 char *buf)
3575 {
3576 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3577
3578 return sysfs_emit(buf, "%s\n", ctrl->subsys->subnqn);
3579 }
3580 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3581
nvme_sysfs_show_hostnqn(struct device * dev,struct device_attribute * attr,char * buf)3582 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3583 struct device_attribute *attr,
3584 char *buf)
3585 {
3586 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3587
3588 return sysfs_emit(buf, "%s\n", ctrl->opts->host->nqn);
3589 }
3590 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3591
nvme_sysfs_show_hostid(struct device * dev,struct device_attribute * attr,char * buf)3592 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3593 struct device_attribute *attr,
3594 char *buf)
3595 {
3596 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3597
3598 return sysfs_emit(buf, "%pU\n", &ctrl->opts->host->id);
3599 }
3600 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3601
nvme_sysfs_show_address(struct device * dev,struct device_attribute * attr,char * buf)3602 static ssize_t nvme_sysfs_show_address(struct device *dev,
3603 struct device_attribute *attr,
3604 char *buf)
3605 {
3606 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3607
3608 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3609 }
3610 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3611
nvme_ctrl_loss_tmo_show(struct device * dev,struct device_attribute * attr,char * buf)3612 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3613 struct device_attribute *attr, char *buf)
3614 {
3615 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3616 struct nvmf_ctrl_options *opts = ctrl->opts;
3617
3618 if (ctrl->opts->max_reconnects == -1)
3619 return sysfs_emit(buf, "off\n");
3620 return sysfs_emit(buf, "%d\n",
3621 opts->max_reconnects * opts->reconnect_delay);
3622 }
3623
nvme_ctrl_loss_tmo_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3624 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3625 struct device_attribute *attr, const char *buf, size_t count)
3626 {
3627 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3628 struct nvmf_ctrl_options *opts = ctrl->opts;
3629 int ctrl_loss_tmo, err;
3630
3631 err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3632 if (err)
3633 return -EINVAL;
3634
3635 if (ctrl_loss_tmo < 0)
3636 opts->max_reconnects = -1;
3637 else
3638 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3639 opts->reconnect_delay);
3640 return count;
3641 }
3642 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3643 nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3644
nvme_ctrl_reconnect_delay_show(struct device * dev,struct device_attribute * attr,char * buf)3645 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3646 struct device_attribute *attr, char *buf)
3647 {
3648 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3649
3650 if (ctrl->opts->reconnect_delay == -1)
3651 return sysfs_emit(buf, "off\n");
3652 return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
3653 }
3654
nvme_ctrl_reconnect_delay_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3655 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3656 struct device_attribute *attr, const char *buf, size_t count)
3657 {
3658 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3659 unsigned int v;
3660 int err;
3661
3662 err = kstrtou32(buf, 10, &v);
3663 if (err)
3664 return err;
3665
3666 ctrl->opts->reconnect_delay = v;
3667 return count;
3668 }
3669 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3670 nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3671
nvme_ctrl_fast_io_fail_tmo_show(struct device * dev,struct device_attribute * attr,char * buf)3672 static ssize_t nvme_ctrl_fast_io_fail_tmo_show(struct device *dev,
3673 struct device_attribute *attr, char *buf)
3674 {
3675 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3676
3677 if (ctrl->opts->fast_io_fail_tmo == -1)
3678 return sysfs_emit(buf, "off\n");
3679 return sysfs_emit(buf, "%d\n", ctrl->opts->fast_io_fail_tmo);
3680 }
3681
nvme_ctrl_fast_io_fail_tmo_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3682 static ssize_t nvme_ctrl_fast_io_fail_tmo_store(struct device *dev,
3683 struct device_attribute *attr, const char *buf, size_t count)
3684 {
3685 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3686 struct nvmf_ctrl_options *opts = ctrl->opts;
3687 int fast_io_fail_tmo, err;
3688
3689 err = kstrtoint(buf, 10, &fast_io_fail_tmo);
3690 if (err)
3691 return -EINVAL;
3692
3693 if (fast_io_fail_tmo < 0)
3694 opts->fast_io_fail_tmo = -1;
3695 else
3696 opts->fast_io_fail_tmo = fast_io_fail_tmo;
3697 return count;
3698 }
3699 static DEVICE_ATTR(fast_io_fail_tmo, S_IRUGO | S_IWUSR,
3700 nvme_ctrl_fast_io_fail_tmo_show, nvme_ctrl_fast_io_fail_tmo_store);
3701
cntrltype_show(struct device * dev,struct device_attribute * attr,char * buf)3702 static ssize_t cntrltype_show(struct device *dev,
3703 struct device_attribute *attr, char *buf)
3704 {
3705 static const char * const type[] = {
3706 [NVME_CTRL_IO] = "io\n",
3707 [NVME_CTRL_DISC] = "discovery\n",
3708 [NVME_CTRL_ADMIN] = "admin\n",
3709 };
3710 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3711
3712 if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
3713 return sysfs_emit(buf, "reserved\n");
3714
3715 return sysfs_emit(buf, type[ctrl->cntrltype]);
3716 }
3717 static DEVICE_ATTR_RO(cntrltype);
3718
dctype_show(struct device * dev,struct device_attribute * attr,char * buf)3719 static ssize_t dctype_show(struct device *dev,
3720 struct device_attribute *attr, char *buf)
3721 {
3722 static const char * const type[] = {
3723 [NVME_DCTYPE_NOT_REPORTED] = "none\n",
3724 [NVME_DCTYPE_DDC] = "ddc\n",
3725 [NVME_DCTYPE_CDC] = "cdc\n",
3726 };
3727 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3728
3729 if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
3730 return sysfs_emit(buf, "reserved\n");
3731
3732 return sysfs_emit(buf, type[ctrl->dctype]);
3733 }
3734 static DEVICE_ATTR_RO(dctype);
3735
3736 #ifdef CONFIG_NVME_AUTH
nvme_ctrl_dhchap_secret_show(struct device * dev,struct device_attribute * attr,char * buf)3737 static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev,
3738 struct device_attribute *attr, char *buf)
3739 {
3740 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3741 struct nvmf_ctrl_options *opts = ctrl->opts;
3742
3743 if (!opts->dhchap_secret)
3744 return sysfs_emit(buf, "none\n");
3745 return sysfs_emit(buf, "%s\n", opts->dhchap_secret);
3746 }
3747
nvme_ctrl_dhchap_secret_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3748 static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev,
3749 struct device_attribute *attr, const char *buf, size_t count)
3750 {
3751 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3752 struct nvmf_ctrl_options *opts = ctrl->opts;
3753 char *dhchap_secret;
3754
3755 if (!ctrl->opts->dhchap_secret)
3756 return -EINVAL;
3757 if (count < 7)
3758 return -EINVAL;
3759 if (memcmp(buf, "DHHC-1:", 7))
3760 return -EINVAL;
3761
3762 dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3763 if (!dhchap_secret)
3764 return -ENOMEM;
3765 memcpy(dhchap_secret, buf, count);
3766 nvme_auth_stop(ctrl);
3767 if (strcmp(dhchap_secret, opts->dhchap_secret)) {
3768 struct nvme_dhchap_key *key, *host_key;
3769 int ret;
3770
3771 ret = nvme_auth_generate_key(dhchap_secret, &key);
3772 if (ret)
3773 return ret;
3774 kfree(opts->dhchap_secret);
3775 opts->dhchap_secret = dhchap_secret;
3776 host_key = ctrl->host_key;
3777 ctrl->host_key = key;
3778 nvme_auth_free_key(host_key);
3779 /* Key has changed; re-authentication with new key */
3780 nvme_auth_reset(ctrl);
3781 }
3782 /* Start re-authentication */
3783 dev_info(ctrl->device, "re-authenticating controller\n");
3784 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3785
3786 return count;
3787 }
3788 static DEVICE_ATTR(dhchap_secret, S_IRUGO | S_IWUSR,
3789 nvme_ctrl_dhchap_secret_show, nvme_ctrl_dhchap_secret_store);
3790
nvme_ctrl_dhchap_ctrl_secret_show(struct device * dev,struct device_attribute * attr,char * buf)3791 static ssize_t nvme_ctrl_dhchap_ctrl_secret_show(struct device *dev,
3792 struct device_attribute *attr, char *buf)
3793 {
3794 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3795 struct nvmf_ctrl_options *opts = ctrl->opts;
3796
3797 if (!opts->dhchap_ctrl_secret)
3798 return sysfs_emit(buf, "none\n");
3799 return sysfs_emit(buf, "%s\n", opts->dhchap_ctrl_secret);
3800 }
3801
nvme_ctrl_dhchap_ctrl_secret_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3802 static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev,
3803 struct device_attribute *attr, const char *buf, size_t count)
3804 {
3805 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3806 struct nvmf_ctrl_options *opts = ctrl->opts;
3807 char *dhchap_secret;
3808
3809 if (!ctrl->opts->dhchap_ctrl_secret)
3810 return -EINVAL;
3811 if (count < 7)
3812 return -EINVAL;
3813 if (memcmp(buf, "DHHC-1:", 7))
3814 return -EINVAL;
3815
3816 dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3817 if (!dhchap_secret)
3818 return -ENOMEM;
3819 memcpy(dhchap_secret, buf, count);
3820 nvme_auth_stop(ctrl);
3821 if (strcmp(dhchap_secret, opts->dhchap_ctrl_secret)) {
3822 struct nvme_dhchap_key *key, *ctrl_key;
3823 int ret;
3824
3825 ret = nvme_auth_generate_key(dhchap_secret, &key);
3826 if (ret)
3827 return ret;
3828 kfree(opts->dhchap_ctrl_secret);
3829 opts->dhchap_ctrl_secret = dhchap_secret;
3830 ctrl_key = ctrl->ctrl_key;
3831 ctrl->ctrl_key = key;
3832 nvme_auth_free_key(ctrl_key);
3833 /* Key has changed; re-authentication with new key */
3834 nvme_auth_reset(ctrl);
3835 }
3836 /* Start re-authentication */
3837 dev_info(ctrl->device, "re-authenticating controller\n");
3838 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3839
3840 return count;
3841 }
3842 static DEVICE_ATTR(dhchap_ctrl_secret, S_IRUGO | S_IWUSR,
3843 nvme_ctrl_dhchap_ctrl_secret_show, nvme_ctrl_dhchap_ctrl_secret_store);
3844 #endif
3845
3846 static struct attribute *nvme_dev_attrs[] = {
3847 &dev_attr_reset_controller.attr,
3848 &dev_attr_rescan_controller.attr,
3849 &dev_attr_model.attr,
3850 &dev_attr_serial.attr,
3851 &dev_attr_firmware_rev.attr,
3852 &dev_attr_cntlid.attr,
3853 &dev_attr_delete_controller.attr,
3854 &dev_attr_transport.attr,
3855 &dev_attr_subsysnqn.attr,
3856 &dev_attr_address.attr,
3857 &dev_attr_state.attr,
3858 &dev_attr_numa_node.attr,
3859 &dev_attr_queue_count.attr,
3860 &dev_attr_sqsize.attr,
3861 &dev_attr_hostnqn.attr,
3862 &dev_attr_hostid.attr,
3863 &dev_attr_ctrl_loss_tmo.attr,
3864 &dev_attr_reconnect_delay.attr,
3865 &dev_attr_fast_io_fail_tmo.attr,
3866 &dev_attr_kato.attr,
3867 &dev_attr_cntrltype.attr,
3868 &dev_attr_dctype.attr,
3869 #ifdef CONFIG_NVME_AUTH
3870 &dev_attr_dhchap_secret.attr,
3871 &dev_attr_dhchap_ctrl_secret.attr,
3872 #endif
3873 NULL
3874 };
3875
nvme_dev_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)3876 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3877 struct attribute *a, int n)
3878 {
3879 struct device *dev = container_of(kobj, struct device, kobj);
3880 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3881
3882 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3883 return 0;
3884 if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3885 return 0;
3886 if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3887 return 0;
3888 if (a == &dev_attr_hostid.attr && !ctrl->opts)
3889 return 0;
3890 if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3891 return 0;
3892 if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3893 return 0;
3894 if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts)
3895 return 0;
3896 #ifdef CONFIG_NVME_AUTH
3897 if (a == &dev_attr_dhchap_secret.attr && !ctrl->opts)
3898 return 0;
3899 if (a == &dev_attr_dhchap_ctrl_secret.attr && !ctrl->opts)
3900 return 0;
3901 #endif
3902
3903 return a->mode;
3904 }
3905
3906 const struct attribute_group nvme_dev_attrs_group = {
3907 .attrs = nvme_dev_attrs,
3908 .is_visible = nvme_dev_attrs_are_visible,
3909 };
3910 EXPORT_SYMBOL_GPL(nvme_dev_attrs_group);
3911
3912 static const struct attribute_group *nvme_dev_attr_groups[] = {
3913 &nvme_dev_attrs_group,
3914 NULL,
3915 };
3916
nvme_find_ns_head(struct nvme_ctrl * ctrl,unsigned nsid)3917 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3918 unsigned nsid)
3919 {
3920 struct nvme_ns_head *h;
3921
3922 lockdep_assert_held(&ctrl->subsys->lock);
3923
3924 list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3925 /*
3926 * Private namespaces can share NSIDs under some conditions.
3927 * In that case we can't use the same ns_head for namespaces
3928 * with the same NSID.
3929 */
3930 if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3931 continue;
3932 if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3933 return h;
3934 }
3935
3936 return NULL;
3937 }
3938
nvme_subsys_check_duplicate_ids(struct nvme_subsystem * subsys,struct nvme_ns_ids * ids)3939 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3940 struct nvme_ns_ids *ids)
3941 {
3942 bool has_uuid = !uuid_is_null(&ids->uuid);
3943 bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3944 bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
3945 struct nvme_ns_head *h;
3946
3947 lockdep_assert_held(&subsys->lock);
3948
3949 list_for_each_entry(h, &subsys->nsheads, entry) {
3950 if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3951 return -EINVAL;
3952 if (has_nguid &&
3953 memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3954 return -EINVAL;
3955 if (has_eui64 &&
3956 memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
3957 return -EINVAL;
3958 }
3959
3960 return 0;
3961 }
3962
nvme_cdev_rel(struct device * dev)3963 static void nvme_cdev_rel(struct device *dev)
3964 {
3965 ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
3966 }
3967
nvme_cdev_del(struct cdev * cdev,struct device * cdev_device)3968 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
3969 {
3970 cdev_device_del(cdev, cdev_device);
3971 put_device(cdev_device);
3972 }
3973
nvme_cdev_add(struct cdev * cdev,struct device * cdev_device,const struct file_operations * fops,struct module * owner)3974 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
3975 const struct file_operations *fops, struct module *owner)
3976 {
3977 int minor, ret;
3978
3979 minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
3980 if (minor < 0)
3981 return minor;
3982 cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
3983 cdev_device->class = nvme_ns_chr_class;
3984 cdev_device->release = nvme_cdev_rel;
3985 device_initialize(cdev_device);
3986 cdev_init(cdev, fops);
3987 cdev->owner = owner;
3988 ret = cdev_device_add(cdev, cdev_device);
3989 if (ret)
3990 put_device(cdev_device);
3991
3992 return ret;
3993 }
3994
nvme_ns_chr_open(struct inode * inode,struct file * file)3995 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
3996 {
3997 return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
3998 }
3999
nvme_ns_chr_release(struct inode * inode,struct file * file)4000 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
4001 {
4002 nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
4003 return 0;
4004 }
4005
4006 static const struct file_operations nvme_ns_chr_fops = {
4007 .owner = THIS_MODULE,
4008 .open = nvme_ns_chr_open,
4009 .release = nvme_ns_chr_release,
4010 .unlocked_ioctl = nvme_ns_chr_ioctl,
4011 .compat_ioctl = compat_ptr_ioctl,
4012 .uring_cmd = nvme_ns_chr_uring_cmd,
4013 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
4014 };
4015
nvme_add_ns_cdev(struct nvme_ns * ns)4016 static int nvme_add_ns_cdev(struct nvme_ns *ns)
4017 {
4018 int ret;
4019
4020 ns->cdev_device.parent = ns->ctrl->device;
4021 ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
4022 ns->ctrl->instance, ns->head->instance);
4023 if (ret)
4024 return ret;
4025
4026 return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
4027 ns->ctrl->ops->module);
4028 }
4029
nvme_alloc_ns_head(struct nvme_ctrl * ctrl,struct nvme_ns_info * info)4030 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
4031 struct nvme_ns_info *info)
4032 {
4033 struct nvme_ns_head *head;
4034 size_t size = sizeof(*head);
4035 int ret = -ENOMEM;
4036
4037 #ifdef CONFIG_NVME_MULTIPATH
4038 size += num_possible_nodes() * sizeof(struct nvme_ns *);
4039 #endif
4040
4041 head = kzalloc(size, GFP_KERNEL);
4042 if (!head)
4043 goto out;
4044 ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
4045 if (ret < 0)
4046 goto out_free_head;
4047 head->instance = ret;
4048 INIT_LIST_HEAD(&head->list);
4049 ret = init_srcu_struct(&head->srcu);
4050 if (ret)
4051 goto out_ida_remove;
4052 head->subsys = ctrl->subsys;
4053 head->ns_id = info->nsid;
4054 head->ids = info->ids;
4055 head->shared = info->is_shared;
4056 kref_init(&head->ref);
4057
4058 if (head->ids.csi) {
4059 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
4060 if (ret)
4061 goto out_cleanup_srcu;
4062 } else
4063 head->effects = ctrl->effects;
4064
4065 ret = nvme_mpath_alloc_disk(ctrl, head);
4066 if (ret)
4067 goto out_cleanup_srcu;
4068
4069 list_add_tail(&head->entry, &ctrl->subsys->nsheads);
4070
4071 kref_get(&ctrl->subsys->ref);
4072
4073 return head;
4074 out_cleanup_srcu:
4075 cleanup_srcu_struct(&head->srcu);
4076 out_ida_remove:
4077 ida_free(&ctrl->subsys->ns_ida, head->instance);
4078 out_free_head:
4079 kfree(head);
4080 out:
4081 if (ret > 0)
4082 ret = blk_status_to_errno(nvme_error_status(ret));
4083 return ERR_PTR(ret);
4084 }
4085
nvme_global_check_duplicate_ids(struct nvme_subsystem * this,struct nvme_ns_ids * ids)4086 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
4087 struct nvme_ns_ids *ids)
4088 {
4089 struct nvme_subsystem *s;
4090 int ret = 0;
4091
4092 /*
4093 * Note that this check is racy as we try to avoid holding the global
4094 * lock over the whole ns_head creation. But it is only intended as
4095 * a sanity check anyway.
4096 */
4097 mutex_lock(&nvme_subsystems_lock);
4098 list_for_each_entry(s, &nvme_subsystems, entry) {
4099 if (s == this)
4100 continue;
4101 mutex_lock(&s->lock);
4102 ret = nvme_subsys_check_duplicate_ids(s, ids);
4103 mutex_unlock(&s->lock);
4104 if (ret)
4105 break;
4106 }
4107 mutex_unlock(&nvme_subsystems_lock);
4108
4109 return ret;
4110 }
4111
nvme_init_ns_head(struct nvme_ns * ns,struct nvme_ns_info * info)4112 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
4113 {
4114 struct nvme_ctrl *ctrl = ns->ctrl;
4115 struct nvme_ns_head *head = NULL;
4116 int ret;
4117
4118 ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
4119 if (ret) {
4120 dev_err(ctrl->device,
4121 "globally duplicate IDs for nsid %d\n", info->nsid);
4122 nvme_print_device_info(ctrl);
4123 return ret;
4124 }
4125
4126 mutex_lock(&ctrl->subsys->lock);
4127 head = nvme_find_ns_head(ctrl, info->nsid);
4128 if (!head) {
4129 ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids);
4130 if (ret) {
4131 dev_err(ctrl->device,
4132 "duplicate IDs in subsystem for nsid %d\n",
4133 info->nsid);
4134 goto out_unlock;
4135 }
4136 head = nvme_alloc_ns_head(ctrl, info);
4137 if (IS_ERR(head)) {
4138 ret = PTR_ERR(head);
4139 goto out_unlock;
4140 }
4141 } else {
4142 ret = -EINVAL;
4143 if (!info->is_shared || !head->shared) {
4144 dev_err(ctrl->device,
4145 "Duplicate unshared namespace %d\n",
4146 info->nsid);
4147 goto out_put_ns_head;
4148 }
4149 if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
4150 dev_err(ctrl->device,
4151 "IDs don't match for shared namespace %d\n",
4152 info->nsid);
4153 goto out_put_ns_head;
4154 }
4155
4156 if (!multipath && !list_empty(&head->list)) {
4157 dev_warn(ctrl->device,
4158 "Found shared namespace %d, but multipathing not supported.\n",
4159 info->nsid);
4160 dev_warn_once(ctrl->device,
4161 "Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0\n.");
4162 }
4163 }
4164
4165 list_add_tail_rcu(&ns->siblings, &head->list);
4166 ns->head = head;
4167 mutex_unlock(&ctrl->subsys->lock);
4168 return 0;
4169
4170 out_put_ns_head:
4171 nvme_put_ns_head(head);
4172 out_unlock:
4173 mutex_unlock(&ctrl->subsys->lock);
4174 return ret;
4175 }
4176
nvme_find_get_ns(struct nvme_ctrl * ctrl,unsigned nsid)4177 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4178 {
4179 struct nvme_ns *ns, *ret = NULL;
4180
4181 down_read(&ctrl->namespaces_rwsem);
4182 list_for_each_entry(ns, &ctrl->namespaces, list) {
4183 if (ns->head->ns_id == nsid) {
4184 if (!nvme_get_ns(ns))
4185 continue;
4186 ret = ns;
4187 break;
4188 }
4189 if (ns->head->ns_id > nsid)
4190 break;
4191 }
4192 up_read(&ctrl->namespaces_rwsem);
4193 return ret;
4194 }
4195 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
4196
4197 /*
4198 * Add the namespace to the controller list while keeping the list ordered.
4199 */
nvme_ns_add_to_ctrl_list(struct nvme_ns * ns)4200 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
4201 {
4202 struct nvme_ns *tmp;
4203
4204 list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
4205 if (tmp->head->ns_id < ns->head->ns_id) {
4206 list_add(&ns->list, &tmp->list);
4207 return;
4208 }
4209 }
4210 list_add(&ns->list, &ns->ctrl->namespaces);
4211 }
4212
nvme_alloc_ns(struct nvme_ctrl * ctrl,struct nvme_ns_info * info)4213 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
4214 {
4215 struct nvme_ns *ns;
4216 struct gendisk *disk;
4217 int node = ctrl->numa_node;
4218
4219 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
4220 if (!ns)
4221 return;
4222
4223 disk = blk_mq_alloc_disk(ctrl->tagset, ns);
4224 if (IS_ERR(disk))
4225 goto out_free_ns;
4226 disk->fops = &nvme_bdev_ops;
4227 disk->private_data = ns;
4228
4229 ns->disk = disk;
4230 ns->queue = disk->queue;
4231
4232 if (ctrl->opts && ctrl->opts->data_digest)
4233 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
4234
4235 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
4236 if (ctrl->ops->supports_pci_p2pdma &&
4237 ctrl->ops->supports_pci_p2pdma(ctrl))
4238 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
4239
4240 ns->ctrl = ctrl;
4241 kref_init(&ns->kref);
4242
4243 if (nvme_init_ns_head(ns, info))
4244 goto out_cleanup_disk;
4245
4246 /*
4247 * If multipathing is enabled, the device name for all disks and not
4248 * just those that represent shared namespaces needs to be based on the
4249 * subsystem instance. Using the controller instance for private
4250 * namespaces could lead to naming collisions between shared and private
4251 * namespaces if they don't use a common numbering scheme.
4252 *
4253 * If multipathing is not enabled, disk names must use the controller
4254 * instance as shared namespaces will show up as multiple block
4255 * devices.
4256 */
4257 if (ns->head->disk) {
4258 sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
4259 ctrl->instance, ns->head->instance);
4260 disk->flags |= GENHD_FL_HIDDEN;
4261 } else if (multipath) {
4262 sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
4263 ns->head->instance);
4264 } else {
4265 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
4266 ns->head->instance);
4267 }
4268
4269 if (nvme_update_ns_info(ns, info))
4270 goto out_unlink_ns;
4271
4272 down_write(&ctrl->namespaces_rwsem);
4273 nvme_ns_add_to_ctrl_list(ns);
4274 up_write(&ctrl->namespaces_rwsem);
4275 nvme_get_ctrl(ctrl);
4276
4277 if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups))
4278 goto out_cleanup_ns_from_list;
4279
4280 if (!nvme_ns_head_multipath(ns->head))
4281 nvme_add_ns_cdev(ns);
4282
4283 nvme_mpath_add_disk(ns, info->anagrpid);
4284 nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
4285
4286 return;
4287
4288 out_cleanup_ns_from_list:
4289 nvme_put_ctrl(ctrl);
4290 down_write(&ctrl->namespaces_rwsem);
4291 list_del_init(&ns->list);
4292 up_write(&ctrl->namespaces_rwsem);
4293 out_unlink_ns:
4294 mutex_lock(&ctrl->subsys->lock);
4295 list_del_rcu(&ns->siblings);
4296 if (list_empty(&ns->head->list))
4297 list_del_init(&ns->head->entry);
4298 mutex_unlock(&ctrl->subsys->lock);
4299 nvme_put_ns_head(ns->head);
4300 out_cleanup_disk:
4301 put_disk(disk);
4302 out_free_ns:
4303 kfree(ns);
4304 }
4305
nvme_ns_remove(struct nvme_ns * ns)4306 static void nvme_ns_remove(struct nvme_ns *ns)
4307 {
4308 bool last_path = false;
4309
4310 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4311 return;
4312
4313 clear_bit(NVME_NS_READY, &ns->flags);
4314 set_capacity(ns->disk, 0);
4315 nvme_fault_inject_fini(&ns->fault_inject);
4316
4317 /*
4318 * Ensure that !NVME_NS_READY is seen by other threads to prevent
4319 * this ns going back into current_path.
4320 */
4321 synchronize_srcu(&ns->head->srcu);
4322
4323 /* wait for concurrent submissions */
4324 if (nvme_mpath_clear_current_path(ns))
4325 synchronize_srcu(&ns->head->srcu);
4326
4327 mutex_lock(&ns->ctrl->subsys->lock);
4328 list_del_rcu(&ns->siblings);
4329 if (list_empty(&ns->head->list)) {
4330 list_del_init(&ns->head->entry);
4331 last_path = true;
4332 }
4333 mutex_unlock(&ns->ctrl->subsys->lock);
4334
4335 /* guarantee not available in head->list */
4336 synchronize_srcu(&ns->head->srcu);
4337
4338 if (!nvme_ns_head_multipath(ns->head))
4339 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
4340 del_gendisk(ns->disk);
4341
4342 down_write(&ns->ctrl->namespaces_rwsem);
4343 list_del_init(&ns->list);
4344 up_write(&ns->ctrl->namespaces_rwsem);
4345
4346 if (last_path)
4347 nvme_mpath_shutdown_disk(ns->head);
4348 nvme_put_ns(ns);
4349 }
4350
nvme_ns_remove_by_nsid(struct nvme_ctrl * ctrl,u32 nsid)4351 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4352 {
4353 struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4354
4355 if (ns) {
4356 nvme_ns_remove(ns);
4357 nvme_put_ns(ns);
4358 }
4359 }
4360
nvme_validate_ns(struct nvme_ns * ns,struct nvme_ns_info * info)4361 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info)
4362 {
4363 int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4364
4365 if (test_bit(NVME_NS_DEAD, &ns->flags))
4366 goto out;
4367
4368 ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4369 if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) {
4370 dev_err(ns->ctrl->device,
4371 "identifiers changed for nsid %d\n", ns->head->ns_id);
4372 goto out;
4373 }
4374
4375 ret = nvme_update_ns_info(ns, info);
4376 out:
4377 /*
4378 * Only remove the namespace if we got a fatal error back from the
4379 * device, otherwise ignore the error and just move on.
4380 *
4381 * TODO: we should probably schedule a delayed retry here.
4382 */
4383 if (ret > 0 && (ret & NVME_SC_DNR))
4384 nvme_ns_remove(ns);
4385 }
4386
nvme_scan_ns(struct nvme_ctrl * ctrl,unsigned nsid)4387 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4388 {
4389 struct nvme_ns_info info = { .nsid = nsid };
4390 struct nvme_ns *ns;
4391
4392 if (nvme_identify_ns_descs(ctrl, &info))
4393 return;
4394
4395 if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) {
4396 dev_warn(ctrl->device,
4397 "command set not reported for nsid: %d\n", nsid);
4398 return;
4399 }
4400
4401 /*
4402 * If available try to use the Command Set Idependent Identify Namespace
4403 * data structure to find all the generic information that is needed to
4404 * set up a namespace. If not fall back to the legacy version.
4405 */
4406 if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) ||
4407 (info.ids.csi != NVME_CSI_NVM && info.ids.csi != NVME_CSI_ZNS)) {
4408 if (nvme_ns_info_from_id_cs_indep(ctrl, &info))
4409 return;
4410 } else {
4411 if (nvme_ns_info_from_identify(ctrl, &info))
4412 return;
4413 }
4414
4415 /*
4416 * Ignore the namespace if it is not ready. We will get an AEN once it
4417 * becomes ready and restart the scan.
4418 */
4419 if (!info.is_ready)
4420 return;
4421
4422 ns = nvme_find_get_ns(ctrl, nsid);
4423 if (ns) {
4424 nvme_validate_ns(ns, &info);
4425 nvme_put_ns(ns);
4426 } else {
4427 nvme_alloc_ns(ctrl, &info);
4428 }
4429 }
4430
nvme_remove_invalid_namespaces(struct nvme_ctrl * ctrl,unsigned nsid)4431 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4432 unsigned nsid)
4433 {
4434 struct nvme_ns *ns, *next;
4435 LIST_HEAD(rm_list);
4436
4437 down_write(&ctrl->namespaces_rwsem);
4438 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4439 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4440 list_move_tail(&ns->list, &rm_list);
4441 }
4442 up_write(&ctrl->namespaces_rwsem);
4443
4444 list_for_each_entry_safe(ns, next, &rm_list, list)
4445 nvme_ns_remove(ns);
4446
4447 }
4448
nvme_scan_ns_list(struct nvme_ctrl * ctrl)4449 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4450 {
4451 const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4452 __le32 *ns_list;
4453 u32 prev = 0;
4454 int ret = 0, i;
4455
4456 if (nvme_ctrl_limited_cns(ctrl))
4457 return -EOPNOTSUPP;
4458
4459 ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4460 if (!ns_list)
4461 return -ENOMEM;
4462
4463 for (;;) {
4464 struct nvme_command cmd = {
4465 .identify.opcode = nvme_admin_identify,
4466 .identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST,
4467 .identify.nsid = cpu_to_le32(prev),
4468 };
4469
4470 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4471 NVME_IDENTIFY_DATA_SIZE);
4472 if (ret) {
4473 dev_warn(ctrl->device,
4474 "Identify NS List failed (status=0x%x)\n", ret);
4475 goto free;
4476 }
4477
4478 for (i = 0; i < nr_entries; i++) {
4479 u32 nsid = le32_to_cpu(ns_list[i]);
4480
4481 if (!nsid) /* end of the list? */
4482 goto out;
4483 nvme_scan_ns(ctrl, nsid);
4484 while (++prev < nsid)
4485 nvme_ns_remove_by_nsid(ctrl, prev);
4486 }
4487 }
4488 out:
4489 nvme_remove_invalid_namespaces(ctrl, prev);
4490 free:
4491 kfree(ns_list);
4492 return ret;
4493 }
4494
nvme_scan_ns_sequential(struct nvme_ctrl * ctrl)4495 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4496 {
4497 struct nvme_id_ctrl *id;
4498 u32 nn, i;
4499
4500 if (nvme_identify_ctrl(ctrl, &id))
4501 return;
4502 nn = le32_to_cpu(id->nn);
4503 kfree(id);
4504
4505 for (i = 1; i <= nn; i++)
4506 nvme_scan_ns(ctrl, i);
4507
4508 nvme_remove_invalid_namespaces(ctrl, nn);
4509 }
4510
nvme_clear_changed_ns_log(struct nvme_ctrl * ctrl)4511 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4512 {
4513 size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4514 __le32 *log;
4515 int error;
4516
4517 log = kzalloc(log_size, GFP_KERNEL);
4518 if (!log)
4519 return;
4520
4521 /*
4522 * We need to read the log to clear the AEN, but we don't want to rely
4523 * on it for the changed namespace information as userspace could have
4524 * raced with us in reading the log page, which could cause us to miss
4525 * updates.
4526 */
4527 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4528 NVME_CSI_NVM, log, log_size, 0);
4529 if (error)
4530 dev_warn(ctrl->device,
4531 "reading changed ns log failed: %d\n", error);
4532
4533 kfree(log);
4534 }
4535
nvme_scan_work(struct work_struct * work)4536 static void nvme_scan_work(struct work_struct *work)
4537 {
4538 struct nvme_ctrl *ctrl =
4539 container_of(work, struct nvme_ctrl, scan_work);
4540 int ret;
4541
4542 /* No tagset on a live ctrl means IO queues could not created */
4543 if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4544 return;
4545
4546 /*
4547 * Identify controller limits can change at controller reset due to
4548 * new firmware download, even though it is not common we cannot ignore
4549 * such scenario. Controller's non-mdts limits are reported in the unit
4550 * of logical blocks that is dependent on the format of attached
4551 * namespace. Hence re-read the limits at the time of ns allocation.
4552 */
4553 ret = nvme_init_non_mdts_limits(ctrl);
4554 if (ret < 0) {
4555 dev_warn(ctrl->device,
4556 "reading non-mdts-limits failed: %d\n", ret);
4557 return;
4558 }
4559
4560 if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4561 dev_info(ctrl->device, "rescanning namespaces.\n");
4562 nvme_clear_changed_ns_log(ctrl);
4563 }
4564
4565 mutex_lock(&ctrl->scan_lock);
4566 if (nvme_scan_ns_list(ctrl) != 0)
4567 nvme_scan_ns_sequential(ctrl);
4568 mutex_unlock(&ctrl->scan_lock);
4569 }
4570
4571 /*
4572 * This function iterates the namespace list unlocked to allow recovery from
4573 * controller failure. It is up to the caller to ensure the namespace list is
4574 * not modified by scan work while this function is executing.
4575 */
nvme_remove_namespaces(struct nvme_ctrl * ctrl)4576 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4577 {
4578 struct nvme_ns *ns, *next;
4579 LIST_HEAD(ns_list);
4580
4581 /*
4582 * make sure to requeue I/O to all namespaces as these
4583 * might result from the scan itself and must complete
4584 * for the scan_work to make progress
4585 */
4586 nvme_mpath_clear_ctrl_paths(ctrl);
4587
4588 /* prevent racing with ns scanning */
4589 flush_work(&ctrl->scan_work);
4590
4591 /*
4592 * The dead states indicates the controller was not gracefully
4593 * disconnected. In that case, we won't be able to flush any data while
4594 * removing the namespaces' disks; fail all the queues now to avoid
4595 * potentially having to clean up the failed sync later.
4596 */
4597 if (ctrl->state == NVME_CTRL_DEAD)
4598 nvme_kill_queues(ctrl);
4599
4600 /* this is a no-op when called from the controller reset handler */
4601 nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4602
4603 down_write(&ctrl->namespaces_rwsem);
4604 list_splice_init(&ctrl->namespaces, &ns_list);
4605 up_write(&ctrl->namespaces_rwsem);
4606
4607 list_for_each_entry_safe(ns, next, &ns_list, list)
4608 nvme_ns_remove(ns);
4609 }
4610 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4611
nvme_class_uevent(struct device * dev,struct kobj_uevent_env * env)4612 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4613 {
4614 struct nvme_ctrl *ctrl =
4615 container_of(dev, struct nvme_ctrl, ctrl_device);
4616 struct nvmf_ctrl_options *opts = ctrl->opts;
4617 int ret;
4618
4619 ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4620 if (ret)
4621 return ret;
4622
4623 if (opts) {
4624 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4625 if (ret)
4626 return ret;
4627
4628 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4629 opts->trsvcid ?: "none");
4630 if (ret)
4631 return ret;
4632
4633 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4634 opts->host_traddr ?: "none");
4635 if (ret)
4636 return ret;
4637
4638 ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4639 opts->host_iface ?: "none");
4640 }
4641 return ret;
4642 }
4643
nvme_change_uevent(struct nvme_ctrl * ctrl,char * envdata)4644 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4645 {
4646 char *envp[2] = { envdata, NULL };
4647
4648 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4649 }
4650
nvme_aen_uevent(struct nvme_ctrl * ctrl)4651 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4652 {
4653 char *envp[2] = { NULL, NULL };
4654 u32 aen_result = ctrl->aen_result;
4655
4656 ctrl->aen_result = 0;
4657 if (!aen_result)
4658 return;
4659
4660 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4661 if (!envp[0])
4662 return;
4663 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4664 kfree(envp[0]);
4665 }
4666
nvme_async_event_work(struct work_struct * work)4667 static void nvme_async_event_work(struct work_struct *work)
4668 {
4669 struct nvme_ctrl *ctrl =
4670 container_of(work, struct nvme_ctrl, async_event_work);
4671
4672 nvme_aen_uevent(ctrl);
4673
4674 /*
4675 * The transport drivers must guarantee AER submission here is safe by
4676 * flushing ctrl async_event_work after changing the controller state
4677 * from LIVE and before freeing the admin queue.
4678 */
4679 if (ctrl->state == NVME_CTRL_LIVE)
4680 ctrl->ops->submit_async_event(ctrl);
4681 }
4682
nvme_ctrl_pp_status(struct nvme_ctrl * ctrl)4683 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4684 {
4685
4686 u32 csts;
4687
4688 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4689 return false;
4690
4691 if (csts == ~0)
4692 return false;
4693
4694 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4695 }
4696
nvme_get_fw_slot_info(struct nvme_ctrl * ctrl)4697 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4698 {
4699 struct nvme_fw_slot_info_log *log;
4700
4701 log = kmalloc(sizeof(*log), GFP_KERNEL);
4702 if (!log)
4703 return;
4704
4705 if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4706 log, sizeof(*log), 0))
4707 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4708 kfree(log);
4709 }
4710
nvme_fw_act_work(struct work_struct * work)4711 static void nvme_fw_act_work(struct work_struct *work)
4712 {
4713 struct nvme_ctrl *ctrl = container_of(work,
4714 struct nvme_ctrl, fw_act_work);
4715 unsigned long fw_act_timeout;
4716
4717 if (ctrl->mtfa)
4718 fw_act_timeout = jiffies +
4719 msecs_to_jiffies(ctrl->mtfa * 100);
4720 else
4721 fw_act_timeout = jiffies +
4722 msecs_to_jiffies(admin_timeout * 1000);
4723
4724 nvme_stop_queues(ctrl);
4725 while (nvme_ctrl_pp_status(ctrl)) {
4726 if (time_after(jiffies, fw_act_timeout)) {
4727 dev_warn(ctrl->device,
4728 "Fw activation timeout, reset controller\n");
4729 nvme_try_sched_reset(ctrl);
4730 return;
4731 }
4732 msleep(100);
4733 }
4734
4735 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4736 return;
4737
4738 nvme_start_queues(ctrl);
4739 /* read FW slot information to clear the AER */
4740 nvme_get_fw_slot_info(ctrl);
4741
4742 queue_work(nvme_wq, &ctrl->async_event_work);
4743 }
4744
nvme_aer_type(u32 result)4745 static u32 nvme_aer_type(u32 result)
4746 {
4747 return result & 0x7;
4748 }
4749
nvme_aer_subtype(u32 result)4750 static u32 nvme_aer_subtype(u32 result)
4751 {
4752 return (result & 0xff00) >> 8;
4753 }
4754
nvme_handle_aen_notice(struct nvme_ctrl * ctrl,u32 result)4755 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4756 {
4757 u32 aer_notice_type = nvme_aer_subtype(result);
4758 bool requeue = true;
4759
4760 trace_nvme_async_event(ctrl, aer_notice_type);
4761
4762 switch (aer_notice_type) {
4763 case NVME_AER_NOTICE_NS_CHANGED:
4764 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4765 nvme_queue_scan(ctrl);
4766 break;
4767 case NVME_AER_NOTICE_FW_ACT_STARTING:
4768 /*
4769 * We are (ab)using the RESETTING state to prevent subsequent
4770 * recovery actions from interfering with the controller's
4771 * firmware activation.
4772 */
4773 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
4774 nvme_auth_stop(ctrl);
4775 requeue = false;
4776 queue_work(nvme_wq, &ctrl->fw_act_work);
4777 }
4778 break;
4779 #ifdef CONFIG_NVME_MULTIPATH
4780 case NVME_AER_NOTICE_ANA:
4781 if (!ctrl->ana_log_buf)
4782 break;
4783 queue_work(nvme_wq, &ctrl->ana_work);
4784 break;
4785 #endif
4786 case NVME_AER_NOTICE_DISC_CHANGED:
4787 ctrl->aen_result = result;
4788 break;
4789 default:
4790 dev_warn(ctrl->device, "async event result %08x\n", result);
4791 }
4792 return requeue;
4793 }
4794
nvme_handle_aer_persistent_error(struct nvme_ctrl * ctrl)4795 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4796 {
4797 trace_nvme_async_event(ctrl, NVME_AER_ERROR);
4798 dev_warn(ctrl->device, "resetting controller due to AER\n");
4799 nvme_reset_ctrl(ctrl);
4800 }
4801
nvme_complete_async_event(struct nvme_ctrl * ctrl,__le16 status,volatile union nvme_result * res)4802 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4803 volatile union nvme_result *res)
4804 {
4805 u32 result = le32_to_cpu(res->u32);
4806 u32 aer_type = nvme_aer_type(result);
4807 u32 aer_subtype = nvme_aer_subtype(result);
4808 bool requeue = true;
4809
4810 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4811 return;
4812
4813 switch (aer_type) {
4814 case NVME_AER_NOTICE:
4815 requeue = nvme_handle_aen_notice(ctrl, result);
4816 break;
4817 case NVME_AER_ERROR:
4818 /*
4819 * For a persistent internal error, don't run async_event_work
4820 * to submit a new AER. The controller reset will do it.
4821 */
4822 if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4823 nvme_handle_aer_persistent_error(ctrl);
4824 return;
4825 }
4826 fallthrough;
4827 case NVME_AER_SMART:
4828 case NVME_AER_CSS:
4829 case NVME_AER_VS:
4830 trace_nvme_async_event(ctrl, aer_type);
4831 ctrl->aen_result = result;
4832 break;
4833 default:
4834 break;
4835 }
4836
4837 if (requeue)
4838 queue_work(nvme_wq, &ctrl->async_event_work);
4839 }
4840 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4841
nvme_alloc_admin_tag_set(struct nvme_ctrl * ctrl,struct blk_mq_tag_set * set,const struct blk_mq_ops * ops,unsigned int cmd_size)4842 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4843 const struct blk_mq_ops *ops, unsigned int cmd_size)
4844 {
4845 int ret;
4846
4847 memset(set, 0, sizeof(*set));
4848 set->ops = ops;
4849 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
4850 if (ctrl->ops->flags & NVME_F_FABRICS)
4851 set->reserved_tags = NVMF_RESERVED_TAGS;
4852 set->numa_node = ctrl->numa_node;
4853 set->flags = BLK_MQ_F_NO_SCHED;
4854 if (ctrl->ops->flags & NVME_F_BLOCKING)
4855 set->flags |= BLK_MQ_F_BLOCKING;
4856 set->cmd_size = cmd_size;
4857 set->driver_data = ctrl;
4858 set->nr_hw_queues = 1;
4859 set->timeout = NVME_ADMIN_TIMEOUT;
4860 ret = blk_mq_alloc_tag_set(set);
4861 if (ret)
4862 return ret;
4863
4864 ctrl->admin_q = blk_mq_init_queue(set);
4865 if (IS_ERR(ctrl->admin_q)) {
4866 ret = PTR_ERR(ctrl->admin_q);
4867 goto out_free_tagset;
4868 }
4869
4870 if (ctrl->ops->flags & NVME_F_FABRICS) {
4871 ctrl->fabrics_q = blk_mq_init_queue(set);
4872 if (IS_ERR(ctrl->fabrics_q)) {
4873 ret = PTR_ERR(ctrl->fabrics_q);
4874 goto out_cleanup_admin_q;
4875 }
4876 }
4877
4878 ctrl->admin_tagset = set;
4879 return 0;
4880
4881 out_cleanup_admin_q:
4882 blk_mq_destroy_queue(ctrl->admin_q);
4883 out_free_tagset:
4884 blk_mq_free_tag_set(ctrl->admin_tagset);
4885 return ret;
4886 }
4887 EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set);
4888
nvme_remove_admin_tag_set(struct nvme_ctrl * ctrl)4889 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
4890 {
4891 blk_mq_destroy_queue(ctrl->admin_q);
4892 if (ctrl->ops->flags & NVME_F_FABRICS)
4893 blk_mq_destroy_queue(ctrl->fabrics_q);
4894 blk_mq_free_tag_set(ctrl->admin_tagset);
4895 }
4896 EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
4897
nvme_alloc_io_tag_set(struct nvme_ctrl * ctrl,struct blk_mq_tag_set * set,const struct blk_mq_ops * ops,unsigned int nr_maps,unsigned int cmd_size)4898 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4899 const struct blk_mq_ops *ops, unsigned int nr_maps,
4900 unsigned int cmd_size)
4901 {
4902 int ret;
4903
4904 memset(set, 0, sizeof(*set));
4905 set->ops = ops;
4906 set->queue_depth = ctrl->sqsize + 1;
4907 set->reserved_tags = NVMF_RESERVED_TAGS;
4908 set->numa_node = ctrl->numa_node;
4909 set->flags = BLK_MQ_F_SHOULD_MERGE;
4910 if (ctrl->ops->flags & NVME_F_BLOCKING)
4911 set->flags |= BLK_MQ_F_BLOCKING;
4912 set->cmd_size = cmd_size,
4913 set->driver_data = ctrl;
4914 set->nr_hw_queues = ctrl->queue_count - 1;
4915 set->timeout = NVME_IO_TIMEOUT;
4916 set->nr_maps = nr_maps;
4917 ret = blk_mq_alloc_tag_set(set);
4918 if (ret)
4919 return ret;
4920
4921 if (ctrl->ops->flags & NVME_F_FABRICS) {
4922 ctrl->connect_q = blk_mq_init_queue(set);
4923 if (IS_ERR(ctrl->connect_q)) {
4924 ret = PTR_ERR(ctrl->connect_q);
4925 goto out_free_tag_set;
4926 }
4927 }
4928
4929 ctrl->tagset = set;
4930 return 0;
4931
4932 out_free_tag_set:
4933 blk_mq_free_tag_set(set);
4934 return ret;
4935 }
4936 EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set);
4937
nvme_remove_io_tag_set(struct nvme_ctrl * ctrl)4938 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)
4939 {
4940 if (ctrl->ops->flags & NVME_F_FABRICS)
4941 blk_mq_destroy_queue(ctrl->connect_q);
4942 blk_mq_free_tag_set(ctrl->tagset);
4943 }
4944 EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
4945
nvme_stop_ctrl(struct nvme_ctrl * ctrl)4946 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4947 {
4948 nvme_mpath_stop(ctrl);
4949 nvme_auth_stop(ctrl);
4950 nvme_stop_keep_alive(ctrl);
4951 nvme_stop_failfast_work(ctrl);
4952 flush_work(&ctrl->async_event_work);
4953 cancel_work_sync(&ctrl->fw_act_work);
4954 if (ctrl->ops->stop_ctrl)
4955 ctrl->ops->stop_ctrl(ctrl);
4956 }
4957 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4958
nvme_start_ctrl(struct nvme_ctrl * ctrl)4959 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4960 {
4961 nvme_start_keep_alive(ctrl);
4962
4963 nvme_enable_aen(ctrl);
4964
4965 /*
4966 * persistent discovery controllers need to send indication to userspace
4967 * to re-read the discovery log page to learn about possible changes
4968 * that were missed. We identify persistent discovery controllers by
4969 * checking that they started once before, hence are reconnecting back.
4970 */
4971 if (test_and_set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
4972 nvme_discovery_ctrl(ctrl))
4973 nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
4974
4975 if (ctrl->queue_count > 1) {
4976 nvme_queue_scan(ctrl);
4977 nvme_start_queues(ctrl);
4978 nvme_mpath_update(ctrl);
4979 }
4980
4981 nvme_change_uevent(ctrl, "NVME_EVENT=connected");
4982 }
4983 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4984
nvme_uninit_ctrl(struct nvme_ctrl * ctrl)4985 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4986 {
4987 nvme_hwmon_exit(ctrl);
4988 nvme_fault_inject_fini(&ctrl->fault_inject);
4989 dev_pm_qos_hide_latency_tolerance(ctrl->device);
4990 cdev_device_del(&ctrl->cdev, ctrl->device);
4991 nvme_put_ctrl(ctrl);
4992 }
4993 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4994
nvme_free_cels(struct nvme_ctrl * ctrl)4995 static void nvme_free_cels(struct nvme_ctrl *ctrl)
4996 {
4997 struct nvme_effects_log *cel;
4998 unsigned long i;
4999
5000 xa_for_each(&ctrl->cels, i, cel) {
5001 xa_erase(&ctrl->cels, i);
5002 kfree(cel);
5003 }
5004
5005 xa_destroy(&ctrl->cels);
5006 }
5007
nvme_free_ctrl(struct device * dev)5008 static void nvme_free_ctrl(struct device *dev)
5009 {
5010 struct nvme_ctrl *ctrl =
5011 container_of(dev, struct nvme_ctrl, ctrl_device);
5012 struct nvme_subsystem *subsys = ctrl->subsys;
5013
5014 if (!subsys || ctrl->instance != subsys->instance)
5015 ida_free(&nvme_instance_ida, ctrl->instance);
5016
5017 nvme_free_cels(ctrl);
5018 nvme_mpath_uninit(ctrl);
5019 nvme_auth_stop(ctrl);
5020 nvme_auth_free(ctrl);
5021 __free_page(ctrl->discard_page);
5022
5023 if (subsys) {
5024 mutex_lock(&nvme_subsystems_lock);
5025 list_del(&ctrl->subsys_entry);
5026 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
5027 mutex_unlock(&nvme_subsystems_lock);
5028 }
5029
5030 ctrl->ops->free_ctrl(ctrl);
5031
5032 if (subsys)
5033 nvme_put_subsystem(subsys);
5034 }
5035
5036 /*
5037 * Initialize a NVMe controller structures. This needs to be called during
5038 * earliest initialization so that we have the initialized structured around
5039 * during probing.
5040 */
nvme_init_ctrl(struct nvme_ctrl * ctrl,struct device * dev,const struct nvme_ctrl_ops * ops,unsigned long quirks)5041 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
5042 const struct nvme_ctrl_ops *ops, unsigned long quirks)
5043 {
5044 int ret;
5045
5046 ctrl->state = NVME_CTRL_NEW;
5047 clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
5048 spin_lock_init(&ctrl->lock);
5049 mutex_init(&ctrl->scan_lock);
5050 INIT_LIST_HEAD(&ctrl->namespaces);
5051 xa_init(&ctrl->cels);
5052 init_rwsem(&ctrl->namespaces_rwsem);
5053 ctrl->dev = dev;
5054 ctrl->ops = ops;
5055 ctrl->quirks = quirks;
5056 ctrl->numa_node = NUMA_NO_NODE;
5057 INIT_WORK(&ctrl->scan_work, nvme_scan_work);
5058 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
5059 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
5060 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
5061 init_waitqueue_head(&ctrl->state_wq);
5062
5063 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
5064 INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
5065 memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
5066 ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
5067
5068 BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
5069 PAGE_SIZE);
5070 ctrl->discard_page = alloc_page(GFP_KERNEL);
5071 if (!ctrl->discard_page) {
5072 ret = -ENOMEM;
5073 goto out;
5074 }
5075
5076 ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
5077 if (ret < 0)
5078 goto out;
5079 ctrl->instance = ret;
5080
5081 device_initialize(&ctrl->ctrl_device);
5082 ctrl->device = &ctrl->ctrl_device;
5083 ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
5084 ctrl->instance);
5085 ctrl->device->class = nvme_class;
5086 ctrl->device->parent = ctrl->dev;
5087 if (ops->dev_attr_groups)
5088 ctrl->device->groups = ops->dev_attr_groups;
5089 else
5090 ctrl->device->groups = nvme_dev_attr_groups;
5091 ctrl->device->release = nvme_free_ctrl;
5092 dev_set_drvdata(ctrl->device, ctrl);
5093 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
5094 if (ret)
5095 goto out_release_instance;
5096
5097 nvme_get_ctrl(ctrl);
5098 cdev_init(&ctrl->cdev, &nvme_dev_fops);
5099 ctrl->cdev.owner = ops->module;
5100 ret = cdev_device_add(&ctrl->cdev, ctrl->device);
5101 if (ret)
5102 goto out_free_name;
5103
5104 /*
5105 * Initialize latency tolerance controls. The sysfs files won't
5106 * be visible to userspace unless the device actually supports APST.
5107 */
5108 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
5109 dev_pm_qos_update_user_latency_tolerance(ctrl->device,
5110 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
5111
5112 nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
5113 nvme_mpath_init_ctrl(ctrl);
5114 nvme_auth_init_ctrl(ctrl);
5115
5116 return 0;
5117 out_free_name:
5118 nvme_put_ctrl(ctrl);
5119 kfree_const(ctrl->device->kobj.name);
5120 out_release_instance:
5121 ida_free(&nvme_instance_ida, ctrl->instance);
5122 out:
5123 if (ctrl->discard_page)
5124 __free_page(ctrl->discard_page);
5125 return ret;
5126 }
5127 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
5128
nvme_start_ns_queue(struct nvme_ns * ns)5129 static void nvme_start_ns_queue(struct nvme_ns *ns)
5130 {
5131 if (test_and_clear_bit(NVME_NS_STOPPED, &ns->flags))
5132 blk_mq_unquiesce_queue(ns->queue);
5133 }
5134
nvme_stop_ns_queue(struct nvme_ns * ns)5135 static void nvme_stop_ns_queue(struct nvme_ns *ns)
5136 {
5137 if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
5138 blk_mq_quiesce_queue(ns->queue);
5139 else
5140 blk_mq_wait_quiesce_done(ns->queue);
5141 }
5142
5143 /*
5144 * Prepare a queue for teardown.
5145 *
5146 * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
5147 * the capacity to 0 after that to avoid blocking dispatchers that may be
5148 * holding bd_butex. This will end buffered writers dirtying pages that can't
5149 * be synced.
5150 */
nvme_set_queue_dying(struct nvme_ns * ns)5151 static void nvme_set_queue_dying(struct nvme_ns *ns)
5152 {
5153 if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
5154 return;
5155
5156 blk_mark_disk_dead(ns->disk);
5157 nvme_start_ns_queue(ns);
5158
5159 set_capacity_and_notify(ns->disk, 0);
5160 }
5161
5162 /**
5163 * nvme_kill_queues(): Ends all namespace queues
5164 * @ctrl: the dead controller that needs to end
5165 *
5166 * Call this function when the driver determines it is unable to get the
5167 * controller in a state capable of servicing IO.
5168 */
nvme_kill_queues(struct nvme_ctrl * ctrl)5169 void nvme_kill_queues(struct nvme_ctrl *ctrl)
5170 {
5171 struct nvme_ns *ns;
5172
5173 down_read(&ctrl->namespaces_rwsem);
5174
5175 /* Forcibly unquiesce queues to avoid blocking dispatch */
5176 if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
5177 nvme_start_admin_queue(ctrl);
5178
5179 list_for_each_entry(ns, &ctrl->namespaces, list)
5180 nvme_set_queue_dying(ns);
5181
5182 up_read(&ctrl->namespaces_rwsem);
5183 }
5184 EXPORT_SYMBOL_GPL(nvme_kill_queues);
5185
nvme_unfreeze(struct nvme_ctrl * ctrl)5186 void nvme_unfreeze(struct nvme_ctrl *ctrl)
5187 {
5188 struct nvme_ns *ns;
5189
5190 down_read(&ctrl->namespaces_rwsem);
5191 list_for_each_entry(ns, &ctrl->namespaces, list)
5192 blk_mq_unfreeze_queue(ns->queue);
5193 up_read(&ctrl->namespaces_rwsem);
5194 }
5195 EXPORT_SYMBOL_GPL(nvme_unfreeze);
5196
nvme_wait_freeze_timeout(struct nvme_ctrl * ctrl,long timeout)5197 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
5198 {
5199 struct nvme_ns *ns;
5200
5201 down_read(&ctrl->namespaces_rwsem);
5202 list_for_each_entry(ns, &ctrl->namespaces, list) {
5203 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
5204 if (timeout <= 0)
5205 break;
5206 }
5207 up_read(&ctrl->namespaces_rwsem);
5208 return timeout;
5209 }
5210 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
5211
nvme_wait_freeze(struct nvme_ctrl * ctrl)5212 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
5213 {
5214 struct nvme_ns *ns;
5215
5216 down_read(&ctrl->namespaces_rwsem);
5217 list_for_each_entry(ns, &ctrl->namespaces, list)
5218 blk_mq_freeze_queue_wait(ns->queue);
5219 up_read(&ctrl->namespaces_rwsem);
5220 }
5221 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
5222
nvme_start_freeze(struct nvme_ctrl * ctrl)5223 void nvme_start_freeze(struct nvme_ctrl *ctrl)
5224 {
5225 struct nvme_ns *ns;
5226
5227 down_read(&ctrl->namespaces_rwsem);
5228 list_for_each_entry(ns, &ctrl->namespaces, list)
5229 blk_freeze_queue_start(ns->queue);
5230 up_read(&ctrl->namespaces_rwsem);
5231 }
5232 EXPORT_SYMBOL_GPL(nvme_start_freeze);
5233
nvme_stop_queues(struct nvme_ctrl * ctrl)5234 void nvme_stop_queues(struct nvme_ctrl *ctrl)
5235 {
5236 struct nvme_ns *ns;
5237
5238 down_read(&ctrl->namespaces_rwsem);
5239 list_for_each_entry(ns, &ctrl->namespaces, list)
5240 nvme_stop_ns_queue(ns);
5241 up_read(&ctrl->namespaces_rwsem);
5242 }
5243 EXPORT_SYMBOL_GPL(nvme_stop_queues);
5244
nvme_start_queues(struct nvme_ctrl * ctrl)5245 void nvme_start_queues(struct nvme_ctrl *ctrl)
5246 {
5247 struct nvme_ns *ns;
5248
5249 down_read(&ctrl->namespaces_rwsem);
5250 list_for_each_entry(ns, &ctrl->namespaces, list)
5251 nvme_start_ns_queue(ns);
5252 up_read(&ctrl->namespaces_rwsem);
5253 }
5254 EXPORT_SYMBOL_GPL(nvme_start_queues);
5255
nvme_stop_admin_queue(struct nvme_ctrl * ctrl)5256 void nvme_stop_admin_queue(struct nvme_ctrl *ctrl)
5257 {
5258 if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5259 blk_mq_quiesce_queue(ctrl->admin_q);
5260 else
5261 blk_mq_wait_quiesce_done(ctrl->admin_q);
5262 }
5263 EXPORT_SYMBOL_GPL(nvme_stop_admin_queue);
5264
nvme_start_admin_queue(struct nvme_ctrl * ctrl)5265 void nvme_start_admin_queue(struct nvme_ctrl *ctrl)
5266 {
5267 if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5268 blk_mq_unquiesce_queue(ctrl->admin_q);
5269 }
5270 EXPORT_SYMBOL_GPL(nvme_start_admin_queue);
5271
nvme_sync_io_queues(struct nvme_ctrl * ctrl)5272 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
5273 {
5274 struct nvme_ns *ns;
5275
5276 down_read(&ctrl->namespaces_rwsem);
5277 list_for_each_entry(ns, &ctrl->namespaces, list)
5278 blk_sync_queue(ns->queue);
5279 up_read(&ctrl->namespaces_rwsem);
5280 }
5281 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
5282
nvme_sync_queues(struct nvme_ctrl * ctrl)5283 void nvme_sync_queues(struct nvme_ctrl *ctrl)
5284 {
5285 nvme_sync_io_queues(ctrl);
5286 if (ctrl->admin_q)
5287 blk_sync_queue(ctrl->admin_q);
5288 }
5289 EXPORT_SYMBOL_GPL(nvme_sync_queues);
5290
nvme_ctrl_from_file(struct file * file)5291 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
5292 {
5293 if (file->f_op != &nvme_dev_fops)
5294 return NULL;
5295 return file->private_data;
5296 }
5297 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
5298
5299 /*
5300 * Check we didn't inadvertently grow the command structure sizes:
5301 */
_nvme_check_size(void)5302 static inline void _nvme_check_size(void)
5303 {
5304 BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
5305 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
5306 BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
5307 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
5308 BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
5309 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
5310 BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
5311 BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
5312 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
5313 BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
5314 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
5315 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
5316 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
5317 BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
5318 NVME_IDENTIFY_DATA_SIZE);
5319 BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
5320 BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
5321 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
5322 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
5323 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
5324 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
5325 BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
5326 BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
5327 BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
5328 }
5329
5330
nvme_core_init(void)5331 static int __init nvme_core_init(void)
5332 {
5333 int result = -ENOMEM;
5334
5335 _nvme_check_size();
5336
5337 nvme_wq = alloc_workqueue("nvme-wq",
5338 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5339 if (!nvme_wq)
5340 goto out;
5341
5342 nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
5343 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5344 if (!nvme_reset_wq)
5345 goto destroy_wq;
5346
5347 nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
5348 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5349 if (!nvme_delete_wq)
5350 goto destroy_reset_wq;
5351
5352 result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
5353 NVME_MINORS, "nvme");
5354 if (result < 0)
5355 goto destroy_delete_wq;
5356
5357 nvme_class = class_create(THIS_MODULE, "nvme");
5358 if (IS_ERR(nvme_class)) {
5359 result = PTR_ERR(nvme_class);
5360 goto unregister_chrdev;
5361 }
5362 nvme_class->dev_uevent = nvme_class_uevent;
5363
5364 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
5365 if (IS_ERR(nvme_subsys_class)) {
5366 result = PTR_ERR(nvme_subsys_class);
5367 goto destroy_class;
5368 }
5369
5370 result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
5371 "nvme-generic");
5372 if (result < 0)
5373 goto destroy_subsys_class;
5374
5375 nvme_ns_chr_class = class_create(THIS_MODULE, "nvme-generic");
5376 if (IS_ERR(nvme_ns_chr_class)) {
5377 result = PTR_ERR(nvme_ns_chr_class);
5378 goto unregister_generic_ns;
5379 }
5380
5381 return 0;
5382
5383 unregister_generic_ns:
5384 unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5385 destroy_subsys_class:
5386 class_destroy(nvme_subsys_class);
5387 destroy_class:
5388 class_destroy(nvme_class);
5389 unregister_chrdev:
5390 unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5391 destroy_delete_wq:
5392 destroy_workqueue(nvme_delete_wq);
5393 destroy_reset_wq:
5394 destroy_workqueue(nvme_reset_wq);
5395 destroy_wq:
5396 destroy_workqueue(nvme_wq);
5397 out:
5398 return result;
5399 }
5400
nvme_core_exit(void)5401 static void __exit nvme_core_exit(void)
5402 {
5403 class_destroy(nvme_ns_chr_class);
5404 class_destroy(nvme_subsys_class);
5405 class_destroy(nvme_class);
5406 unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5407 unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5408 destroy_workqueue(nvme_delete_wq);
5409 destroy_workqueue(nvme_reset_wq);
5410 destroy_workqueue(nvme_wq);
5411 ida_destroy(&nvme_ns_chr_minor_ida);
5412 ida_destroy(&nvme_instance_ida);
5413 }
5414
5415 MODULE_LICENSE("GPL");
5416 MODULE_VERSION("1.0");
5417 module_init(nvme_core_init);
5418 module_exit(nvme_core_exit);
5419