1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Universal Flash Storage Host controller driver Core
4 * Copyright (C) 2011-2013 Samsung India Software Operations
5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
6 *
7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
10 */
11
12 #include <linux/async.h>
13 #include <linux/devfreq.h>
14 #include <linux/nls.h>
15 #include <linux/of.h>
16 #include <linux/bitfield.h>
17 #include <linux/blk-pm.h>
18 #include <linux/blkdev.h>
19 #include <linux/clk.h>
20 #include <linux/delay.h>
21 #include <linux/interrupt.h>
22 #include <linux/module.h>
23 #include <linux/regulator/consumer.h>
24 #include <linux/sched/clock.h>
25 #include <scsi/scsi_cmnd.h>
26 #include <scsi/scsi_dbg.h>
27 #include <scsi/scsi_driver.h>
28 #include <scsi/scsi_eh.h>
29 #include "ufshcd-priv.h"
30 #include <ufs/ufs_quirks.h>
31 #include <ufs/unipro.h>
32 #include "ufs-sysfs.h"
33 #include "ufs-debugfs.h"
34 #include "ufs-fault-injection.h"
35 #include "ufs_bsg.h"
36 #include "ufshcd-crypto.h"
37 #include "ufshpb.h"
38 #include <asm/unaligned.h>
39
40 #define CREATE_TRACE_POINTS
41 #include <trace/events/ufs.h>
42
43 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
44 UTP_TASK_REQ_COMPL |\
45 UFSHCD_ERROR_MASK)
46 /* UIC command timeout, unit: ms */
47 #define UIC_CMD_TIMEOUT 500
48
49 /* NOP OUT retries waiting for NOP IN response */
50 #define NOP_OUT_RETRIES 10
51 /* Timeout after 50 msecs if NOP OUT hangs without response */
52 #define NOP_OUT_TIMEOUT 50 /* msecs */
53
54 /* Query request retries */
55 #define QUERY_REQ_RETRIES 3
56 /* Query request timeout */
57 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
58
59 /* Task management command timeout */
60 #define TM_CMD_TIMEOUT 100 /* msecs */
61
62 /* maximum number of retries for a general UIC command */
63 #define UFS_UIC_COMMAND_RETRIES 3
64
65 /* maximum number of link-startup retries */
66 #define DME_LINKSTARTUP_RETRIES 3
67
68 /* maximum number of reset retries before giving up */
69 #define MAX_HOST_RESET_RETRIES 5
70
71 /* Maximum number of error handler retries before giving up */
72 #define MAX_ERR_HANDLER_RETRIES 5
73
74 /* Expose the flag value from utp_upiu_query.value */
75 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
76
77 /* Interrupt aggregation default timeout, unit: 40us */
78 #define INT_AGGR_DEF_TO 0x02
79
80 /* default delay of autosuspend: 2000 ms */
81 #define RPM_AUTOSUSPEND_DELAY_MS 2000
82
83 /* Default delay of RPM device flush delayed work */
84 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
85
86 /* Default value of wait time before gating device ref clock */
87 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
88
89 /* Polling time to wait for fDeviceInit */
90 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
91
92 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
93 ({ \
94 int _ret; \
95 if (_on) \
96 _ret = ufshcd_enable_vreg(_dev, _vreg); \
97 else \
98 _ret = ufshcd_disable_vreg(_dev, _vreg); \
99 _ret; \
100 })
101
102 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
103 size_t __len = (len); \
104 print_hex_dump(KERN_ERR, prefix_str, \
105 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
106 16, 4, buf, __len, false); \
107 } while (0)
108
ufshcd_dump_regs(struct ufs_hba * hba,size_t offset,size_t len,const char * prefix)109 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
110 const char *prefix)
111 {
112 u32 *regs;
113 size_t pos;
114
115 if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
116 return -EINVAL;
117
118 regs = kzalloc(len, GFP_ATOMIC);
119 if (!regs)
120 return -ENOMEM;
121
122 for (pos = 0; pos < len; pos += 4) {
123 if (offset == 0 &&
124 pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
125 pos <= REG_UIC_ERROR_CODE_DME)
126 continue;
127 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
128 }
129
130 ufshcd_hex_dump(prefix, regs, len);
131 kfree(regs);
132
133 return 0;
134 }
135 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
136
137 enum {
138 UFSHCD_MAX_CHANNEL = 0,
139 UFSHCD_MAX_ID = 1,
140 UFSHCD_NUM_RESERVED = 1,
141 UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED,
142 UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED,
143 };
144
145 static const char *const ufshcd_state_name[] = {
146 [UFSHCD_STATE_RESET] = "reset",
147 [UFSHCD_STATE_OPERATIONAL] = "operational",
148 [UFSHCD_STATE_ERROR] = "error",
149 [UFSHCD_STATE_EH_SCHEDULED_FATAL] = "eh_fatal",
150 [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL] = "eh_non_fatal",
151 };
152
153 /* UFSHCD error handling flags */
154 enum {
155 UFSHCD_EH_IN_PROGRESS = (1 << 0),
156 };
157
158 /* UFSHCD UIC layer error flags */
159 enum {
160 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
161 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
162 UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
163 UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
164 UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
165 UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
166 UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
167 };
168
169 #define ufshcd_set_eh_in_progress(h) \
170 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
171 #define ufshcd_eh_in_progress(h) \
172 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
173 #define ufshcd_clear_eh_in_progress(h) \
174 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
175
176 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
177 [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
178 [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
179 [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
180 [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
181 [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
182 [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
183 /*
184 * For DeepSleep, the link is first put in hibern8 and then off.
185 * Leaving the link in hibern8 is not supported.
186 */
187 [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
188 };
189
190 static inline enum ufs_dev_pwr_mode
ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)191 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
192 {
193 return ufs_pm_lvl_states[lvl].dev_state;
194 }
195
196 static inline enum uic_link_state
ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)197 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
198 {
199 return ufs_pm_lvl_states[lvl].link_state;
200 }
201
202 static inline enum ufs_pm_level
ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,enum uic_link_state link_state)203 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
204 enum uic_link_state link_state)
205 {
206 enum ufs_pm_level lvl;
207
208 for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
209 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
210 (ufs_pm_lvl_states[lvl].link_state == link_state))
211 return lvl;
212 }
213
214 /* if no match found, return the level 0 */
215 return UFS_PM_LVL_0;
216 }
217
218 static const struct ufs_dev_quirk ufs_fixups[] = {
219 /* UFS cards deviations table */
220 { .wmanufacturerid = UFS_VENDOR_MICRON,
221 .model = UFS_ANY_MODEL,
222 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
223 UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ },
224 { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
225 .model = UFS_ANY_MODEL,
226 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
227 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
228 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
229 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
230 .model = UFS_ANY_MODEL,
231 .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
232 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
233 .model = "hB8aL1" /*H28U62301AMR*/,
234 .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
235 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
236 .model = UFS_ANY_MODEL,
237 .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
238 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
239 .model = "THGLF2G9C8KBADG",
240 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
241 { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
242 .model = "THGLF2G9D8KBADG",
243 .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
244 {}
245 };
246
247 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
248 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
249 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
250 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
251 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
252 static void ufshcd_hba_exit(struct ufs_hba *hba);
253 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
254 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
255 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
256 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
257 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
258 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
259 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
260 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
261 static irqreturn_t ufshcd_intr(int irq, void *__hba);
262 static int ufshcd_change_power_mode(struct ufs_hba *hba,
263 struct ufs_pa_layer_attr *pwr_mode);
264 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
265 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
266 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
267 struct ufs_vreg *vreg);
268 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
269 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
270 bool enable);
271 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
272 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
273
ufshcd_enable_irq(struct ufs_hba * hba)274 static inline void ufshcd_enable_irq(struct ufs_hba *hba)
275 {
276 if (!hba->is_irq_enabled) {
277 enable_irq(hba->irq);
278 hba->is_irq_enabled = true;
279 }
280 }
281
ufshcd_disable_irq(struct ufs_hba * hba)282 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
283 {
284 if (hba->is_irq_enabled) {
285 disable_irq(hba->irq);
286 hba->is_irq_enabled = false;
287 }
288 }
289
ufshcd_configure_wb(struct ufs_hba * hba)290 static void ufshcd_configure_wb(struct ufs_hba *hba)
291 {
292 if (!ufshcd_is_wb_allowed(hba))
293 return;
294
295 ufshcd_wb_toggle(hba, true);
296
297 ufshcd_wb_toggle_buf_flush_during_h8(hba, true);
298
299 if (ufshcd_is_wb_buf_flush_allowed(hba))
300 ufshcd_wb_toggle_buf_flush(hba, true);
301 }
302
ufshcd_scsi_unblock_requests(struct ufs_hba * hba)303 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
304 {
305 if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
306 scsi_unblock_requests(hba->host);
307 }
308
ufshcd_scsi_block_requests(struct ufs_hba * hba)309 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
310 {
311 if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
312 scsi_block_requests(hba->host);
313 }
314
ufshcd_add_cmd_upiu_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)315 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
316 enum ufs_trace_str_t str_t)
317 {
318 struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
319 struct utp_upiu_header *header;
320
321 if (!trace_ufshcd_upiu_enabled())
322 return;
323
324 if (str_t == UFS_CMD_SEND)
325 header = &rq->header;
326 else
327 header = &hba->lrb[tag].ucd_rsp_ptr->header;
328
329 trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
330 UFS_TSF_CDB);
331 }
332
ufshcd_add_query_upiu_trace(struct ufs_hba * hba,enum ufs_trace_str_t str_t,struct utp_upiu_req * rq_rsp)333 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
334 enum ufs_trace_str_t str_t,
335 struct utp_upiu_req *rq_rsp)
336 {
337 if (!trace_ufshcd_upiu_enabled())
338 return;
339
340 trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
341 &rq_rsp->qr, UFS_TSF_OSF);
342 }
343
ufshcd_add_tm_upiu_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)344 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
345 enum ufs_trace_str_t str_t)
346 {
347 struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
348
349 if (!trace_ufshcd_upiu_enabled())
350 return;
351
352 if (str_t == UFS_TM_SEND)
353 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
354 &descp->upiu_req.req_header,
355 &descp->upiu_req.input_param1,
356 UFS_TSF_TM_INPUT);
357 else
358 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
359 &descp->upiu_rsp.rsp_header,
360 &descp->upiu_rsp.output_param1,
361 UFS_TSF_TM_OUTPUT);
362 }
363
ufshcd_add_uic_command_trace(struct ufs_hba * hba,const struct uic_command * ucmd,enum ufs_trace_str_t str_t)364 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
365 const struct uic_command *ucmd,
366 enum ufs_trace_str_t str_t)
367 {
368 u32 cmd;
369
370 if (!trace_ufshcd_uic_command_enabled())
371 return;
372
373 if (str_t == UFS_CMD_SEND)
374 cmd = ucmd->command;
375 else
376 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
377
378 trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
379 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
380 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
381 ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
382 }
383
ufshcd_add_command_trace(struct ufs_hba * hba,unsigned int tag,enum ufs_trace_str_t str_t)384 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
385 enum ufs_trace_str_t str_t)
386 {
387 u64 lba = 0;
388 u8 opcode = 0, group_id = 0;
389 u32 intr, doorbell;
390 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
391 struct scsi_cmnd *cmd = lrbp->cmd;
392 struct request *rq = scsi_cmd_to_rq(cmd);
393 int transfer_len = -1;
394
395 if (!cmd)
396 return;
397
398 /* trace UPIU also */
399 ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
400 if (!trace_ufshcd_command_enabled())
401 return;
402
403 opcode = cmd->cmnd[0];
404
405 if (opcode == READ_10 || opcode == WRITE_10) {
406 /*
407 * Currently we only fully trace read(10) and write(10) commands
408 */
409 transfer_len =
410 be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
411 lba = scsi_get_lba(cmd);
412 if (opcode == WRITE_10)
413 group_id = lrbp->cmd->cmnd[6];
414 } else if (opcode == UNMAP) {
415 /*
416 * The number of Bytes to be unmapped beginning with the lba.
417 */
418 transfer_len = blk_rq_bytes(rq);
419 lba = scsi_get_lba(cmd);
420 }
421
422 intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
423 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
424 trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
425 doorbell, transfer_len, intr, lba, opcode, group_id);
426 }
427
ufshcd_print_clk_freqs(struct ufs_hba * hba)428 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
429 {
430 struct ufs_clk_info *clki;
431 struct list_head *head = &hba->clk_list_head;
432
433 if (list_empty(head))
434 return;
435
436 list_for_each_entry(clki, head, list) {
437 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
438 clki->max_freq)
439 dev_err(hba->dev, "clk: %s, rate: %u\n",
440 clki->name, clki->curr_freq);
441 }
442 }
443
ufshcd_print_evt(struct ufs_hba * hba,u32 id,const char * err_name)444 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
445 const char *err_name)
446 {
447 int i;
448 bool found = false;
449 const struct ufs_event_hist *e;
450
451 if (id >= UFS_EVT_CNT)
452 return;
453
454 e = &hba->ufs_stats.event[id];
455
456 for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
457 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
458
459 if (e->tstamp[p] == 0)
460 continue;
461 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
462 e->val[p], div_u64(e->tstamp[p], 1000));
463 found = true;
464 }
465
466 if (!found)
467 dev_err(hba->dev, "No record of %s\n", err_name);
468 else
469 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
470 }
471
ufshcd_print_evt_hist(struct ufs_hba * hba)472 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
473 {
474 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
475
476 ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
477 ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
478 ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
479 ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
480 ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
481 ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
482 "auto_hibern8_err");
483 ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
484 ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
485 "link_startup_fail");
486 ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
487 ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
488 "suspend_fail");
489 ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
490 ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
491 ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
492
493 ufshcd_vops_dbg_register_dump(hba);
494 }
495
496 static
ufshcd_print_trs(struct ufs_hba * hba,unsigned long bitmap,bool pr_prdt)497 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
498 {
499 const struct ufshcd_lrb *lrbp;
500 int prdt_length;
501 int tag;
502
503 for_each_set_bit(tag, &bitmap, hba->nutrs) {
504 lrbp = &hba->lrb[tag];
505
506 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
507 tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
508 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
509 tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
510 dev_err(hba->dev,
511 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
512 tag, (u64)lrbp->utrd_dma_addr);
513
514 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
515 sizeof(struct utp_transfer_req_desc));
516 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
517 (u64)lrbp->ucd_req_dma_addr);
518 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
519 sizeof(struct utp_upiu_req));
520 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
521 (u64)lrbp->ucd_rsp_dma_addr);
522 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
523 sizeof(struct utp_upiu_rsp));
524
525 prdt_length = le16_to_cpu(
526 lrbp->utr_descriptor_ptr->prd_table_length);
527 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
528 prdt_length /= sizeof(struct ufshcd_sg_entry);
529
530 dev_err(hba->dev,
531 "UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
532 tag, prdt_length,
533 (u64)lrbp->ucd_prdt_dma_addr);
534
535 if (pr_prdt)
536 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
537 sizeof(struct ufshcd_sg_entry) * prdt_length);
538 }
539 }
540
ufshcd_print_tmrs(struct ufs_hba * hba,unsigned long bitmap)541 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
542 {
543 int tag;
544
545 for_each_set_bit(tag, &bitmap, hba->nutmrs) {
546 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
547
548 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
549 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
550 }
551 }
552
ufshcd_print_host_state(struct ufs_hba * hba)553 static void ufshcd_print_host_state(struct ufs_hba *hba)
554 {
555 const struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
556
557 dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
558 dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
559 hba->outstanding_reqs, hba->outstanding_tasks);
560 dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
561 hba->saved_err, hba->saved_uic_err);
562 dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
563 hba->curr_dev_pwr_mode, hba->uic_link_state);
564 dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
565 hba->pm_op_in_progress, hba->is_sys_suspended);
566 dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
567 hba->auto_bkops_enabled, hba->host->host_self_blocked);
568 dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
569 dev_err(hba->dev,
570 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
571 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000),
572 hba->ufs_stats.hibern8_exit_cnt);
573 dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
574 div_u64(hba->ufs_stats.last_intr_ts, 1000),
575 hba->ufs_stats.last_intr_status);
576 dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
577 hba->eh_flags, hba->req_abort_count);
578 dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
579 hba->ufs_version, hba->capabilities, hba->caps);
580 dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
581 hba->dev_quirks);
582 if (sdev_ufs)
583 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
584 sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
585
586 ufshcd_print_clk_freqs(hba);
587 }
588
589 /**
590 * ufshcd_print_pwr_info - print power params as saved in hba
591 * power info
592 * @hba: per-adapter instance
593 */
ufshcd_print_pwr_info(struct ufs_hba * hba)594 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
595 {
596 static const char * const names[] = {
597 "INVALID MODE",
598 "FAST MODE",
599 "SLOW_MODE",
600 "INVALID MODE",
601 "FASTAUTO_MODE",
602 "SLOWAUTO_MODE",
603 "INVALID MODE",
604 };
605
606 /*
607 * Using dev_dbg to avoid messages during runtime PM to avoid
608 * never-ending cycles of messages written back to storage by user space
609 * causing runtime resume, causing more messages and so on.
610 */
611 dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
612 __func__,
613 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
614 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
615 names[hba->pwr_info.pwr_rx],
616 names[hba->pwr_info.pwr_tx],
617 hba->pwr_info.hs_rate);
618 }
619
ufshcd_device_reset(struct ufs_hba * hba)620 static void ufshcd_device_reset(struct ufs_hba *hba)
621 {
622 int err;
623
624 err = ufshcd_vops_device_reset(hba);
625
626 if (!err) {
627 ufshcd_set_ufs_dev_active(hba);
628 if (ufshcd_is_wb_allowed(hba)) {
629 hba->dev_info.wb_enabled = false;
630 hba->dev_info.wb_buf_flush_enabled = false;
631 }
632 }
633 if (err != -EOPNOTSUPP)
634 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
635 }
636
ufshcd_delay_us(unsigned long us,unsigned long tolerance)637 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
638 {
639 if (!us)
640 return;
641
642 if (us < 10)
643 udelay(us);
644 else
645 usleep_range(us, us + tolerance);
646 }
647 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
648
649 /**
650 * ufshcd_wait_for_register - wait for register value to change
651 * @hba: per-adapter interface
652 * @reg: mmio register offset
653 * @mask: mask to apply to the read register value
654 * @val: value to wait for
655 * @interval_us: polling interval in microseconds
656 * @timeout_ms: timeout in milliseconds
657 *
658 * Return:
659 * -ETIMEDOUT on error, zero on success.
660 */
ufshcd_wait_for_register(struct ufs_hba * hba,u32 reg,u32 mask,u32 val,unsigned long interval_us,unsigned long timeout_ms)661 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
662 u32 val, unsigned long interval_us,
663 unsigned long timeout_ms)
664 {
665 int err = 0;
666 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
667
668 /* ignore bits that we don't intend to wait on */
669 val = val & mask;
670
671 while ((ufshcd_readl(hba, reg) & mask) != val) {
672 usleep_range(interval_us, interval_us + 50);
673 if (time_after(jiffies, timeout)) {
674 if ((ufshcd_readl(hba, reg) & mask) != val)
675 err = -ETIMEDOUT;
676 break;
677 }
678 }
679
680 return err;
681 }
682
683 /**
684 * ufshcd_get_intr_mask - Get the interrupt bit mask
685 * @hba: Pointer to adapter instance
686 *
687 * Returns interrupt bit mask per version
688 */
ufshcd_get_intr_mask(struct ufs_hba * hba)689 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
690 {
691 if (hba->ufs_version == ufshci_version(1, 0))
692 return INTERRUPT_MASK_ALL_VER_10;
693 if (hba->ufs_version <= ufshci_version(2, 0))
694 return INTERRUPT_MASK_ALL_VER_11;
695
696 return INTERRUPT_MASK_ALL_VER_21;
697 }
698
699 /**
700 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
701 * @hba: Pointer to adapter instance
702 *
703 * Returns UFSHCI version supported by the controller
704 */
ufshcd_get_ufs_version(struct ufs_hba * hba)705 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
706 {
707 u32 ufshci_ver;
708
709 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
710 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
711 else
712 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
713
714 /*
715 * UFSHCI v1.x uses a different version scheme, in order
716 * to allow the use of comparisons with the ufshci_version
717 * function, we convert it to the same scheme as ufs 2.0+.
718 */
719 if (ufshci_ver & 0x00010000)
720 return ufshci_version(1, ufshci_ver & 0x00000100);
721
722 return ufshci_ver;
723 }
724
725 /**
726 * ufshcd_is_device_present - Check if any device connected to
727 * the host controller
728 * @hba: pointer to adapter instance
729 *
730 * Returns true if device present, false if no device detected
731 */
ufshcd_is_device_present(struct ufs_hba * hba)732 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
733 {
734 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
735 }
736
737 /**
738 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
739 * @lrbp: pointer to local command reference block
740 *
741 * This function is used to get the OCS field from UTRD
742 * Returns the OCS field in the UTRD
743 */
ufshcd_get_tr_ocs(struct ufshcd_lrb * lrbp)744 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
745 {
746 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
747 }
748
749 /**
750 * ufshcd_utrl_clear() - Clear requests from the controller request list.
751 * @hba: per adapter instance
752 * @mask: mask with one bit set for each request to be cleared
753 */
ufshcd_utrl_clear(struct ufs_hba * hba,u32 mask)754 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask)
755 {
756 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
757 mask = ~mask;
758 /*
759 * From the UFSHCI specification: "UTP Transfer Request List CLear
760 * Register (UTRLCLR): This field is bit significant. Each bit
761 * corresponds to a slot in the UTP Transfer Request List, where bit 0
762 * corresponds to request slot 0. A bit in this field is set to ‘0’
763 * by host software to indicate to the host controller that a transfer
764 * request slot is cleared. The host controller
765 * shall free up any resources associated to the request slot
766 * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The
767 * host software indicates no change to request slots by setting the
768 * associated bits in this field to ‘1’. Bits in this field shall only
769 * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’."
770 */
771 ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR);
772 }
773
774 /**
775 * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register
776 * @hba: per adapter instance
777 * @pos: position of the bit to be cleared
778 */
ufshcd_utmrl_clear(struct ufs_hba * hba,u32 pos)779 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
780 {
781 if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
782 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
783 else
784 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
785 }
786
787 /**
788 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
789 * @reg: Register value of host controller status
790 *
791 * Returns integer, 0 on Success and positive value if failed
792 */
ufshcd_get_lists_status(u32 reg)793 static inline int ufshcd_get_lists_status(u32 reg)
794 {
795 return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
796 }
797
798 /**
799 * ufshcd_get_uic_cmd_result - Get the UIC command result
800 * @hba: Pointer to adapter instance
801 *
802 * This function gets the result of UIC command completion
803 * Returns 0 on success, non zero value on error
804 */
ufshcd_get_uic_cmd_result(struct ufs_hba * hba)805 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
806 {
807 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
808 MASK_UIC_COMMAND_RESULT;
809 }
810
811 /**
812 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
813 * @hba: Pointer to adapter instance
814 *
815 * This function gets UIC command argument3
816 * Returns 0 on success, non zero value on error
817 */
ufshcd_get_dme_attr_val(struct ufs_hba * hba)818 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
819 {
820 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
821 }
822
823 /**
824 * ufshcd_get_req_rsp - returns the TR response transaction type
825 * @ucd_rsp_ptr: pointer to response UPIU
826 */
827 static inline int
ufshcd_get_req_rsp(struct utp_upiu_rsp * ucd_rsp_ptr)828 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
829 {
830 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
831 }
832
833 /**
834 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
835 * @ucd_rsp_ptr: pointer to response UPIU
836 *
837 * This function gets the response status and scsi_status from response UPIU
838 * Returns the response result code.
839 */
840 static inline int
ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp * ucd_rsp_ptr)841 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
842 {
843 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
844 }
845
846 /*
847 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
848 * from response UPIU
849 * @ucd_rsp_ptr: pointer to response UPIU
850 *
851 * Return the data segment length.
852 */
853 static inline unsigned int
ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp * ucd_rsp_ptr)854 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
855 {
856 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
857 MASK_RSP_UPIU_DATA_SEG_LEN;
858 }
859
860 /**
861 * ufshcd_is_exception_event - Check if the device raised an exception event
862 * @ucd_rsp_ptr: pointer to response UPIU
863 *
864 * The function checks if the device raised an exception event indicated in
865 * the Device Information field of response UPIU.
866 *
867 * Returns true if exception is raised, false otherwise.
868 */
ufshcd_is_exception_event(struct utp_upiu_rsp * ucd_rsp_ptr)869 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
870 {
871 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
872 MASK_RSP_EXCEPTION_EVENT;
873 }
874
875 /**
876 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
877 * @hba: per adapter instance
878 */
879 static inline void
ufshcd_reset_intr_aggr(struct ufs_hba * hba)880 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
881 {
882 ufshcd_writel(hba, INT_AGGR_ENABLE |
883 INT_AGGR_COUNTER_AND_TIMER_RESET,
884 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
885 }
886
887 /**
888 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
889 * @hba: per adapter instance
890 * @cnt: Interrupt aggregation counter threshold
891 * @tmout: Interrupt aggregation timeout value
892 */
893 static inline void
ufshcd_config_intr_aggr(struct ufs_hba * hba,u8 cnt,u8 tmout)894 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
895 {
896 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
897 INT_AGGR_COUNTER_THLD_VAL(cnt) |
898 INT_AGGR_TIMEOUT_VAL(tmout),
899 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
900 }
901
902 /**
903 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
904 * @hba: per adapter instance
905 */
ufshcd_disable_intr_aggr(struct ufs_hba * hba)906 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
907 {
908 ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
909 }
910
911 /**
912 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
913 * When run-stop registers are set to 1, it indicates the
914 * host controller that it can process the requests
915 * @hba: per adapter instance
916 */
ufshcd_enable_run_stop_reg(struct ufs_hba * hba)917 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
918 {
919 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
920 REG_UTP_TASK_REQ_LIST_RUN_STOP);
921 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
922 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
923 }
924
925 /**
926 * ufshcd_hba_start - Start controller initialization sequence
927 * @hba: per adapter instance
928 */
ufshcd_hba_start(struct ufs_hba * hba)929 static inline void ufshcd_hba_start(struct ufs_hba *hba)
930 {
931 u32 val = CONTROLLER_ENABLE;
932
933 if (ufshcd_crypto_enable(hba))
934 val |= CRYPTO_GENERAL_ENABLE;
935
936 ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
937 }
938
939 /**
940 * ufshcd_is_hba_active - Get controller state
941 * @hba: per adapter instance
942 *
943 * Returns true if and only if the controller is active.
944 */
ufshcd_is_hba_active(struct ufs_hba * hba)945 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
946 {
947 return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
948 }
949
ufshcd_get_local_unipro_ver(struct ufs_hba * hba)950 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
951 {
952 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
953 if (hba->ufs_version <= ufshci_version(1, 1))
954 return UFS_UNIPRO_VER_1_41;
955 else
956 return UFS_UNIPRO_VER_1_6;
957 }
958 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
959
ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba * hba)960 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
961 {
962 /*
963 * If both host and device support UniPro ver1.6 or later, PA layer
964 * parameters tuning happens during link startup itself.
965 *
966 * We can manually tune PA layer parameters if either host or device
967 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
968 * logic simple, we will only do manual tuning if local unipro version
969 * doesn't support ver1.6 or later.
970 */
971 return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6;
972 }
973
974 /**
975 * ufshcd_set_clk_freq - set UFS controller clock frequencies
976 * @hba: per adapter instance
977 * @scale_up: If True, set max possible frequency othewise set low frequency
978 *
979 * Returns 0 if successful
980 * Returns < 0 for any other errors
981 */
ufshcd_set_clk_freq(struct ufs_hba * hba,bool scale_up)982 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
983 {
984 int ret = 0;
985 struct ufs_clk_info *clki;
986 struct list_head *head = &hba->clk_list_head;
987
988 if (list_empty(head))
989 goto out;
990
991 list_for_each_entry(clki, head, list) {
992 if (!IS_ERR_OR_NULL(clki->clk)) {
993 if (scale_up && clki->max_freq) {
994 if (clki->curr_freq == clki->max_freq)
995 continue;
996
997 ret = clk_set_rate(clki->clk, clki->max_freq);
998 if (ret) {
999 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1000 __func__, clki->name,
1001 clki->max_freq, ret);
1002 break;
1003 }
1004 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1005 "scaled up", clki->name,
1006 clki->curr_freq,
1007 clki->max_freq);
1008
1009 clki->curr_freq = clki->max_freq;
1010
1011 } else if (!scale_up && clki->min_freq) {
1012 if (clki->curr_freq == clki->min_freq)
1013 continue;
1014
1015 ret = clk_set_rate(clki->clk, clki->min_freq);
1016 if (ret) {
1017 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1018 __func__, clki->name,
1019 clki->min_freq, ret);
1020 break;
1021 }
1022 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1023 "scaled down", clki->name,
1024 clki->curr_freq,
1025 clki->min_freq);
1026 clki->curr_freq = clki->min_freq;
1027 }
1028 }
1029 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1030 clki->name, clk_get_rate(clki->clk));
1031 }
1032
1033 out:
1034 return ret;
1035 }
1036
1037 /**
1038 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1039 * @hba: per adapter instance
1040 * @scale_up: True if scaling up and false if scaling down
1041 *
1042 * Returns 0 if successful
1043 * Returns < 0 for any other errors
1044 */
ufshcd_scale_clks(struct ufs_hba * hba,bool scale_up)1045 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1046 {
1047 int ret = 0;
1048 ktime_t start = ktime_get();
1049
1050 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1051 if (ret)
1052 goto out;
1053
1054 ret = ufshcd_set_clk_freq(hba, scale_up);
1055 if (ret)
1056 goto out;
1057
1058 ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1059 if (ret)
1060 ufshcd_set_clk_freq(hba, !scale_up);
1061
1062 out:
1063 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1064 (scale_up ? "up" : "down"),
1065 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1066 return ret;
1067 }
1068
1069 /**
1070 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1071 * @hba: per adapter instance
1072 * @scale_up: True if scaling up and false if scaling down
1073 *
1074 * Returns true if scaling is required, false otherwise.
1075 */
ufshcd_is_devfreq_scaling_required(struct ufs_hba * hba,bool scale_up)1076 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1077 bool scale_up)
1078 {
1079 struct ufs_clk_info *clki;
1080 struct list_head *head = &hba->clk_list_head;
1081
1082 if (list_empty(head))
1083 return false;
1084
1085 list_for_each_entry(clki, head, list) {
1086 if (!IS_ERR_OR_NULL(clki->clk)) {
1087 if (scale_up && clki->max_freq) {
1088 if (clki->curr_freq == clki->max_freq)
1089 continue;
1090 return true;
1091 } else if (!scale_up && clki->min_freq) {
1092 if (clki->curr_freq == clki->min_freq)
1093 continue;
1094 return true;
1095 }
1096 }
1097 }
1098
1099 return false;
1100 }
1101
1102 /*
1103 * Determine the number of pending commands by counting the bits in the SCSI
1104 * device budget maps. This approach has been selected because a bit is set in
1105 * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1106 * flag. The host_self_blocked flag can be modified by calling
1107 * scsi_block_requests() or scsi_unblock_requests().
1108 */
ufshcd_pending_cmds(struct ufs_hba * hba)1109 static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1110 {
1111 const struct scsi_device *sdev;
1112 u32 pending = 0;
1113
1114 lockdep_assert_held(hba->host->host_lock);
1115 __shost_for_each_device(sdev, hba->host)
1116 pending += sbitmap_weight(&sdev->budget_map);
1117
1118 return pending;
1119 }
1120
ufshcd_wait_for_doorbell_clr(struct ufs_hba * hba,u64 wait_timeout_us)1121 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1122 u64 wait_timeout_us)
1123 {
1124 unsigned long flags;
1125 int ret = 0;
1126 u32 tm_doorbell;
1127 u32 tr_pending;
1128 bool timeout = false, do_last_check = false;
1129 ktime_t start;
1130
1131 ufshcd_hold(hba, false);
1132 spin_lock_irqsave(hba->host->host_lock, flags);
1133 /*
1134 * Wait for all the outstanding tasks/transfer requests.
1135 * Verify by checking the doorbell registers are clear.
1136 */
1137 start = ktime_get();
1138 do {
1139 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1140 ret = -EBUSY;
1141 goto out;
1142 }
1143
1144 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1145 tr_pending = ufshcd_pending_cmds(hba);
1146 if (!tm_doorbell && !tr_pending) {
1147 timeout = false;
1148 break;
1149 } else if (do_last_check) {
1150 break;
1151 }
1152
1153 spin_unlock_irqrestore(hba->host->host_lock, flags);
1154 schedule();
1155 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1156 wait_timeout_us) {
1157 timeout = true;
1158 /*
1159 * We might have scheduled out for long time so make
1160 * sure to check if doorbells are cleared by this time
1161 * or not.
1162 */
1163 do_last_check = true;
1164 }
1165 spin_lock_irqsave(hba->host->host_lock, flags);
1166 } while (tm_doorbell || tr_pending);
1167
1168 if (timeout) {
1169 dev_err(hba->dev,
1170 "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1171 __func__, tm_doorbell, tr_pending);
1172 ret = -EBUSY;
1173 }
1174 out:
1175 spin_unlock_irqrestore(hba->host->host_lock, flags);
1176 ufshcd_release(hba);
1177 return ret;
1178 }
1179
1180 /**
1181 * ufshcd_scale_gear - scale up/down UFS gear
1182 * @hba: per adapter instance
1183 * @scale_up: True for scaling up gear and false for scaling down
1184 *
1185 * Returns 0 for success,
1186 * Returns -EBUSY if scaling can't happen at this time
1187 * Returns non-zero for any other errors
1188 */
ufshcd_scale_gear(struct ufs_hba * hba,bool scale_up)1189 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1190 {
1191 int ret = 0;
1192 struct ufs_pa_layer_attr new_pwr_info;
1193
1194 if (scale_up) {
1195 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1196 sizeof(struct ufs_pa_layer_attr));
1197 } else {
1198 memcpy(&new_pwr_info, &hba->pwr_info,
1199 sizeof(struct ufs_pa_layer_attr));
1200
1201 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1202 hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1203 /* save the current power mode */
1204 memcpy(&hba->clk_scaling.saved_pwr_info.info,
1205 &hba->pwr_info,
1206 sizeof(struct ufs_pa_layer_attr));
1207
1208 /* scale down gear */
1209 new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1210 new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1211 }
1212 }
1213
1214 /* check if the power mode needs to be changed or not? */
1215 ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1216 if (ret)
1217 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1218 __func__, ret,
1219 hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1220 new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1221
1222 return ret;
1223 }
1224
ufshcd_clock_scaling_prepare(struct ufs_hba * hba)1225 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1226 {
1227 #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */
1228 int ret = 0;
1229 /*
1230 * make sure that there are no outstanding requests when
1231 * clock scaling is in progress
1232 */
1233 ufshcd_scsi_block_requests(hba);
1234 mutex_lock(&hba->wb_mutex);
1235 down_write(&hba->clk_scaling_lock);
1236
1237 if (!hba->clk_scaling.is_allowed ||
1238 ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1239 ret = -EBUSY;
1240 up_write(&hba->clk_scaling_lock);
1241 mutex_unlock(&hba->wb_mutex);
1242 ufshcd_scsi_unblock_requests(hba);
1243 goto out;
1244 }
1245
1246 /* let's not get into low power until clock scaling is completed */
1247 ufshcd_hold(hba, false);
1248
1249 out:
1250 return ret;
1251 }
1252
ufshcd_clock_scaling_unprepare(struct ufs_hba * hba,int err,bool scale_up)1253 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up)
1254 {
1255 up_write(&hba->clk_scaling_lock);
1256
1257 /* Enable Write Booster if we have scaled up else disable it */
1258 if (ufshcd_enable_wb_if_scaling_up(hba) && !err)
1259 ufshcd_wb_toggle(hba, scale_up);
1260
1261 mutex_unlock(&hba->wb_mutex);
1262
1263 ufshcd_scsi_unblock_requests(hba);
1264 ufshcd_release(hba);
1265 }
1266
1267 /**
1268 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1269 * @hba: per adapter instance
1270 * @scale_up: True for scaling up and false for scalin down
1271 *
1272 * Returns 0 for success,
1273 * Returns -EBUSY if scaling can't happen at this time
1274 * Returns non-zero for any other errors
1275 */
ufshcd_devfreq_scale(struct ufs_hba * hba,bool scale_up)1276 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1277 {
1278 int ret = 0;
1279
1280 ret = ufshcd_clock_scaling_prepare(hba);
1281 if (ret)
1282 return ret;
1283
1284 /* scale down the gear before scaling down clocks */
1285 if (!scale_up) {
1286 ret = ufshcd_scale_gear(hba, false);
1287 if (ret)
1288 goto out_unprepare;
1289 }
1290
1291 ret = ufshcd_scale_clks(hba, scale_up);
1292 if (ret) {
1293 if (!scale_up)
1294 ufshcd_scale_gear(hba, true);
1295 goto out_unprepare;
1296 }
1297
1298 /* scale up the gear after scaling up clocks */
1299 if (scale_up) {
1300 ret = ufshcd_scale_gear(hba, true);
1301 if (ret) {
1302 ufshcd_scale_clks(hba, false);
1303 goto out_unprepare;
1304 }
1305 }
1306
1307 out_unprepare:
1308 ufshcd_clock_scaling_unprepare(hba, ret, scale_up);
1309 return ret;
1310 }
1311
ufshcd_clk_scaling_suspend_work(struct work_struct * work)1312 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1313 {
1314 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1315 clk_scaling.suspend_work);
1316 unsigned long irq_flags;
1317
1318 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1319 if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1320 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1321 return;
1322 }
1323 hba->clk_scaling.is_suspended = true;
1324 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1325
1326 __ufshcd_suspend_clkscaling(hba);
1327 }
1328
ufshcd_clk_scaling_resume_work(struct work_struct * work)1329 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1330 {
1331 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1332 clk_scaling.resume_work);
1333 unsigned long irq_flags;
1334
1335 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1336 if (!hba->clk_scaling.is_suspended) {
1337 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1338 return;
1339 }
1340 hba->clk_scaling.is_suspended = false;
1341 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1342
1343 devfreq_resume_device(hba->devfreq);
1344 }
1345
ufshcd_devfreq_target(struct device * dev,unsigned long * freq,u32 flags)1346 static int ufshcd_devfreq_target(struct device *dev,
1347 unsigned long *freq, u32 flags)
1348 {
1349 int ret = 0;
1350 struct ufs_hba *hba = dev_get_drvdata(dev);
1351 ktime_t start;
1352 bool scale_up, sched_clk_scaling_suspend_work = false;
1353 struct list_head *clk_list = &hba->clk_list_head;
1354 struct ufs_clk_info *clki;
1355 unsigned long irq_flags;
1356
1357 if (!ufshcd_is_clkscaling_supported(hba))
1358 return -EINVAL;
1359
1360 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1361 /* Override with the closest supported frequency */
1362 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1363 spin_lock_irqsave(hba->host->host_lock, irq_flags);
1364 if (ufshcd_eh_in_progress(hba)) {
1365 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1366 return 0;
1367 }
1368
1369 if (!hba->clk_scaling.active_reqs)
1370 sched_clk_scaling_suspend_work = true;
1371
1372 if (list_empty(clk_list)) {
1373 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1374 goto out;
1375 }
1376
1377 /* Decide based on the rounded-off frequency and update */
1378 scale_up = *freq == clki->max_freq;
1379 if (!scale_up)
1380 *freq = clki->min_freq;
1381 /* Update the frequency */
1382 if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1383 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1384 ret = 0;
1385 goto out; /* no state change required */
1386 }
1387 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1388
1389 start = ktime_get();
1390 ret = ufshcd_devfreq_scale(hba, scale_up);
1391
1392 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1393 (scale_up ? "up" : "down"),
1394 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1395
1396 out:
1397 if (sched_clk_scaling_suspend_work)
1398 queue_work(hba->clk_scaling.workq,
1399 &hba->clk_scaling.suspend_work);
1400
1401 return ret;
1402 }
1403
ufshcd_devfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * stat)1404 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1405 struct devfreq_dev_status *stat)
1406 {
1407 struct ufs_hba *hba = dev_get_drvdata(dev);
1408 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1409 unsigned long flags;
1410 struct list_head *clk_list = &hba->clk_list_head;
1411 struct ufs_clk_info *clki;
1412 ktime_t curr_t;
1413
1414 if (!ufshcd_is_clkscaling_supported(hba))
1415 return -EINVAL;
1416
1417 memset(stat, 0, sizeof(*stat));
1418
1419 spin_lock_irqsave(hba->host->host_lock, flags);
1420 curr_t = ktime_get();
1421 if (!scaling->window_start_t)
1422 goto start_window;
1423
1424 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1425 /*
1426 * If current frequency is 0, then the ondemand governor considers
1427 * there's no initial frequency set. And it always requests to set
1428 * to max. frequency.
1429 */
1430 stat->current_frequency = clki->curr_freq;
1431 if (scaling->is_busy_started)
1432 scaling->tot_busy_t += ktime_us_delta(curr_t,
1433 scaling->busy_start_t);
1434
1435 stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1436 stat->busy_time = scaling->tot_busy_t;
1437 start_window:
1438 scaling->window_start_t = curr_t;
1439 scaling->tot_busy_t = 0;
1440
1441 if (hba->outstanding_reqs) {
1442 scaling->busy_start_t = curr_t;
1443 scaling->is_busy_started = true;
1444 } else {
1445 scaling->busy_start_t = 0;
1446 scaling->is_busy_started = false;
1447 }
1448 spin_unlock_irqrestore(hba->host->host_lock, flags);
1449 return 0;
1450 }
1451
ufshcd_devfreq_init(struct ufs_hba * hba)1452 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1453 {
1454 struct list_head *clk_list = &hba->clk_list_head;
1455 struct ufs_clk_info *clki;
1456 struct devfreq *devfreq;
1457 int ret;
1458
1459 /* Skip devfreq if we don't have any clocks in the list */
1460 if (list_empty(clk_list))
1461 return 0;
1462
1463 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1464 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1465 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1466
1467 ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1468 &hba->vps->ondemand_data);
1469 devfreq = devfreq_add_device(hba->dev,
1470 &hba->vps->devfreq_profile,
1471 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1472 &hba->vps->ondemand_data);
1473 if (IS_ERR(devfreq)) {
1474 ret = PTR_ERR(devfreq);
1475 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1476
1477 dev_pm_opp_remove(hba->dev, clki->min_freq);
1478 dev_pm_opp_remove(hba->dev, clki->max_freq);
1479 return ret;
1480 }
1481
1482 hba->devfreq = devfreq;
1483
1484 return 0;
1485 }
1486
ufshcd_devfreq_remove(struct ufs_hba * hba)1487 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1488 {
1489 struct list_head *clk_list = &hba->clk_list_head;
1490 struct ufs_clk_info *clki;
1491
1492 if (!hba->devfreq)
1493 return;
1494
1495 devfreq_remove_device(hba->devfreq);
1496 hba->devfreq = NULL;
1497
1498 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1499 dev_pm_opp_remove(hba->dev, clki->min_freq);
1500 dev_pm_opp_remove(hba->dev, clki->max_freq);
1501 }
1502
__ufshcd_suspend_clkscaling(struct ufs_hba * hba)1503 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1504 {
1505 unsigned long flags;
1506
1507 devfreq_suspend_device(hba->devfreq);
1508 spin_lock_irqsave(hba->host->host_lock, flags);
1509 hba->clk_scaling.window_start_t = 0;
1510 spin_unlock_irqrestore(hba->host->host_lock, flags);
1511 }
1512
ufshcd_suspend_clkscaling(struct ufs_hba * hba)1513 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1514 {
1515 unsigned long flags;
1516 bool suspend = false;
1517
1518 cancel_work_sync(&hba->clk_scaling.suspend_work);
1519 cancel_work_sync(&hba->clk_scaling.resume_work);
1520
1521 spin_lock_irqsave(hba->host->host_lock, flags);
1522 if (!hba->clk_scaling.is_suspended) {
1523 suspend = true;
1524 hba->clk_scaling.is_suspended = true;
1525 }
1526 spin_unlock_irqrestore(hba->host->host_lock, flags);
1527
1528 if (suspend)
1529 __ufshcd_suspend_clkscaling(hba);
1530 }
1531
ufshcd_resume_clkscaling(struct ufs_hba * hba)1532 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1533 {
1534 unsigned long flags;
1535 bool resume = false;
1536
1537 spin_lock_irqsave(hba->host->host_lock, flags);
1538 if (hba->clk_scaling.is_suspended) {
1539 resume = true;
1540 hba->clk_scaling.is_suspended = false;
1541 }
1542 spin_unlock_irqrestore(hba->host->host_lock, flags);
1543
1544 if (resume)
1545 devfreq_resume_device(hba->devfreq);
1546 }
1547
ufshcd_clkscale_enable_show(struct device * dev,struct device_attribute * attr,char * buf)1548 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1549 struct device_attribute *attr, char *buf)
1550 {
1551 struct ufs_hba *hba = dev_get_drvdata(dev);
1552
1553 return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1554 }
1555
ufshcd_clkscale_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1556 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1557 struct device_attribute *attr, const char *buf, size_t count)
1558 {
1559 struct ufs_hba *hba = dev_get_drvdata(dev);
1560 u32 value;
1561 int err = 0;
1562
1563 if (kstrtou32(buf, 0, &value))
1564 return -EINVAL;
1565
1566 down(&hba->host_sem);
1567 if (!ufshcd_is_user_access_allowed(hba)) {
1568 err = -EBUSY;
1569 goto out;
1570 }
1571
1572 value = !!value;
1573 if (value == hba->clk_scaling.is_enabled)
1574 goto out;
1575
1576 ufshcd_rpm_get_sync(hba);
1577 ufshcd_hold(hba, false);
1578
1579 hba->clk_scaling.is_enabled = value;
1580
1581 if (value) {
1582 ufshcd_resume_clkscaling(hba);
1583 } else {
1584 ufshcd_suspend_clkscaling(hba);
1585 err = ufshcd_devfreq_scale(hba, true);
1586 if (err)
1587 dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1588 __func__, err);
1589 }
1590
1591 ufshcd_release(hba);
1592 ufshcd_rpm_put_sync(hba);
1593 out:
1594 up(&hba->host_sem);
1595 return err ? err : count;
1596 }
1597
ufshcd_init_clk_scaling_sysfs(struct ufs_hba * hba)1598 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1599 {
1600 hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1601 hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1602 sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1603 hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1604 hba->clk_scaling.enable_attr.attr.mode = 0644;
1605 if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1606 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1607 }
1608
ufshcd_remove_clk_scaling_sysfs(struct ufs_hba * hba)1609 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1610 {
1611 if (hba->clk_scaling.enable_attr.attr.name)
1612 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1613 }
1614
ufshcd_init_clk_scaling(struct ufs_hba * hba)1615 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1616 {
1617 char wq_name[sizeof("ufs_clkscaling_00")];
1618
1619 if (!ufshcd_is_clkscaling_supported(hba))
1620 return;
1621
1622 if (!hba->clk_scaling.min_gear)
1623 hba->clk_scaling.min_gear = UFS_HS_G1;
1624
1625 INIT_WORK(&hba->clk_scaling.suspend_work,
1626 ufshcd_clk_scaling_suspend_work);
1627 INIT_WORK(&hba->clk_scaling.resume_work,
1628 ufshcd_clk_scaling_resume_work);
1629
1630 snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1631 hba->host->host_no);
1632 hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1633
1634 hba->clk_scaling.is_initialized = true;
1635 }
1636
ufshcd_exit_clk_scaling(struct ufs_hba * hba)1637 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1638 {
1639 if (!hba->clk_scaling.is_initialized)
1640 return;
1641
1642 ufshcd_remove_clk_scaling_sysfs(hba);
1643 destroy_workqueue(hba->clk_scaling.workq);
1644 ufshcd_devfreq_remove(hba);
1645 hba->clk_scaling.is_initialized = false;
1646 }
1647
ufshcd_ungate_work(struct work_struct * work)1648 static void ufshcd_ungate_work(struct work_struct *work)
1649 {
1650 int ret;
1651 unsigned long flags;
1652 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1653 clk_gating.ungate_work);
1654
1655 cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1656
1657 spin_lock_irqsave(hba->host->host_lock, flags);
1658 if (hba->clk_gating.state == CLKS_ON) {
1659 spin_unlock_irqrestore(hba->host->host_lock, flags);
1660 goto unblock_reqs;
1661 }
1662
1663 spin_unlock_irqrestore(hba->host->host_lock, flags);
1664 ufshcd_hba_vreg_set_hpm(hba);
1665 ufshcd_setup_clocks(hba, true);
1666
1667 ufshcd_enable_irq(hba);
1668
1669 /* Exit from hibern8 */
1670 if (ufshcd_can_hibern8_during_gating(hba)) {
1671 /* Prevent gating in this path */
1672 hba->clk_gating.is_suspended = true;
1673 if (ufshcd_is_link_hibern8(hba)) {
1674 ret = ufshcd_uic_hibern8_exit(hba);
1675 if (ret)
1676 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1677 __func__, ret);
1678 else
1679 ufshcd_set_link_active(hba);
1680 }
1681 hba->clk_gating.is_suspended = false;
1682 }
1683 unblock_reqs:
1684 ufshcd_scsi_unblock_requests(hba);
1685 }
1686
1687 /**
1688 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1689 * Also, exit from hibern8 mode and set the link as active.
1690 * @hba: per adapter instance
1691 * @async: This indicates whether caller should ungate clocks asynchronously.
1692 */
ufshcd_hold(struct ufs_hba * hba,bool async)1693 int ufshcd_hold(struct ufs_hba *hba, bool async)
1694 {
1695 int rc = 0;
1696 bool flush_result;
1697 unsigned long flags;
1698
1699 if (!ufshcd_is_clkgating_allowed(hba) ||
1700 !hba->clk_gating.is_initialized)
1701 goto out;
1702 spin_lock_irqsave(hba->host->host_lock, flags);
1703 hba->clk_gating.active_reqs++;
1704
1705 start:
1706 switch (hba->clk_gating.state) {
1707 case CLKS_ON:
1708 /*
1709 * Wait for the ungate work to complete if in progress.
1710 * Though the clocks may be in ON state, the link could
1711 * still be in hibner8 state if hibern8 is allowed
1712 * during clock gating.
1713 * Make sure we exit hibern8 state also in addition to
1714 * clocks being ON.
1715 */
1716 if (ufshcd_can_hibern8_during_gating(hba) &&
1717 ufshcd_is_link_hibern8(hba)) {
1718 if (async) {
1719 rc = -EAGAIN;
1720 hba->clk_gating.active_reqs--;
1721 break;
1722 }
1723 spin_unlock_irqrestore(hba->host->host_lock, flags);
1724 flush_result = flush_work(&hba->clk_gating.ungate_work);
1725 if (hba->clk_gating.is_suspended && !flush_result)
1726 goto out;
1727 spin_lock_irqsave(hba->host->host_lock, flags);
1728 goto start;
1729 }
1730 break;
1731 case REQ_CLKS_OFF:
1732 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1733 hba->clk_gating.state = CLKS_ON;
1734 trace_ufshcd_clk_gating(dev_name(hba->dev),
1735 hba->clk_gating.state);
1736 break;
1737 }
1738 /*
1739 * If we are here, it means gating work is either done or
1740 * currently running. Hence, fall through to cancel gating
1741 * work and to enable clocks.
1742 */
1743 fallthrough;
1744 case CLKS_OFF:
1745 hba->clk_gating.state = REQ_CLKS_ON;
1746 trace_ufshcd_clk_gating(dev_name(hba->dev),
1747 hba->clk_gating.state);
1748 if (queue_work(hba->clk_gating.clk_gating_workq,
1749 &hba->clk_gating.ungate_work))
1750 ufshcd_scsi_block_requests(hba);
1751 /*
1752 * fall through to check if we should wait for this
1753 * work to be done or not.
1754 */
1755 fallthrough;
1756 case REQ_CLKS_ON:
1757 if (async) {
1758 rc = -EAGAIN;
1759 hba->clk_gating.active_reqs--;
1760 break;
1761 }
1762
1763 spin_unlock_irqrestore(hba->host->host_lock, flags);
1764 flush_work(&hba->clk_gating.ungate_work);
1765 /* Make sure state is CLKS_ON before returning */
1766 spin_lock_irqsave(hba->host->host_lock, flags);
1767 goto start;
1768 default:
1769 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1770 __func__, hba->clk_gating.state);
1771 break;
1772 }
1773 spin_unlock_irqrestore(hba->host->host_lock, flags);
1774 out:
1775 return rc;
1776 }
1777 EXPORT_SYMBOL_GPL(ufshcd_hold);
1778
ufshcd_gate_work(struct work_struct * work)1779 static void ufshcd_gate_work(struct work_struct *work)
1780 {
1781 struct ufs_hba *hba = container_of(work, struct ufs_hba,
1782 clk_gating.gate_work.work);
1783 unsigned long flags;
1784 int ret;
1785
1786 spin_lock_irqsave(hba->host->host_lock, flags);
1787 /*
1788 * In case you are here to cancel this work the gating state
1789 * would be marked as REQ_CLKS_ON. In this case save time by
1790 * skipping the gating work and exit after changing the clock
1791 * state to CLKS_ON.
1792 */
1793 if (hba->clk_gating.is_suspended ||
1794 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1795 hba->clk_gating.state = CLKS_ON;
1796 trace_ufshcd_clk_gating(dev_name(hba->dev),
1797 hba->clk_gating.state);
1798 goto rel_lock;
1799 }
1800
1801 if (hba->clk_gating.active_reqs
1802 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1803 || hba->outstanding_reqs || hba->outstanding_tasks
1804 || hba->active_uic_cmd || hba->uic_async_done)
1805 goto rel_lock;
1806
1807 spin_unlock_irqrestore(hba->host->host_lock, flags);
1808
1809 /* put the link into hibern8 mode before turning off clocks */
1810 if (ufshcd_can_hibern8_during_gating(hba)) {
1811 ret = ufshcd_uic_hibern8_enter(hba);
1812 if (ret) {
1813 hba->clk_gating.state = CLKS_ON;
1814 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1815 __func__, ret);
1816 trace_ufshcd_clk_gating(dev_name(hba->dev),
1817 hba->clk_gating.state);
1818 goto out;
1819 }
1820 ufshcd_set_link_hibern8(hba);
1821 }
1822
1823 ufshcd_disable_irq(hba);
1824
1825 ufshcd_setup_clocks(hba, false);
1826
1827 /* Put the host controller in low power mode if possible */
1828 ufshcd_hba_vreg_set_lpm(hba);
1829 /*
1830 * In case you are here to cancel this work the gating state
1831 * would be marked as REQ_CLKS_ON. In this case keep the state
1832 * as REQ_CLKS_ON which would anyway imply that clocks are off
1833 * and a request to turn them on is pending. By doing this way,
1834 * we keep the state machine in tact and this would ultimately
1835 * prevent from doing cancel work multiple times when there are
1836 * new requests arriving before the current cancel work is done.
1837 */
1838 spin_lock_irqsave(hba->host->host_lock, flags);
1839 if (hba->clk_gating.state == REQ_CLKS_OFF) {
1840 hba->clk_gating.state = CLKS_OFF;
1841 trace_ufshcd_clk_gating(dev_name(hba->dev),
1842 hba->clk_gating.state);
1843 }
1844 rel_lock:
1845 spin_unlock_irqrestore(hba->host->host_lock, flags);
1846 out:
1847 return;
1848 }
1849
1850 /* host lock must be held before calling this variant */
__ufshcd_release(struct ufs_hba * hba)1851 static void __ufshcd_release(struct ufs_hba *hba)
1852 {
1853 if (!ufshcd_is_clkgating_allowed(hba))
1854 return;
1855
1856 hba->clk_gating.active_reqs--;
1857
1858 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1859 hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1860 hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
1861 hba->active_uic_cmd || hba->uic_async_done ||
1862 hba->clk_gating.state == CLKS_OFF)
1863 return;
1864
1865 hba->clk_gating.state = REQ_CLKS_OFF;
1866 trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1867 queue_delayed_work(hba->clk_gating.clk_gating_workq,
1868 &hba->clk_gating.gate_work,
1869 msecs_to_jiffies(hba->clk_gating.delay_ms));
1870 }
1871
ufshcd_release(struct ufs_hba * hba)1872 void ufshcd_release(struct ufs_hba *hba)
1873 {
1874 unsigned long flags;
1875
1876 spin_lock_irqsave(hba->host->host_lock, flags);
1877 __ufshcd_release(hba);
1878 spin_unlock_irqrestore(hba->host->host_lock, flags);
1879 }
1880 EXPORT_SYMBOL_GPL(ufshcd_release);
1881
ufshcd_clkgate_delay_show(struct device * dev,struct device_attribute * attr,char * buf)1882 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1883 struct device_attribute *attr, char *buf)
1884 {
1885 struct ufs_hba *hba = dev_get_drvdata(dev);
1886
1887 return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1888 }
1889
ufshcd_clkgate_delay_set(struct device * dev,unsigned long value)1890 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
1891 {
1892 struct ufs_hba *hba = dev_get_drvdata(dev);
1893 unsigned long flags;
1894
1895 spin_lock_irqsave(hba->host->host_lock, flags);
1896 hba->clk_gating.delay_ms = value;
1897 spin_unlock_irqrestore(hba->host->host_lock, flags);
1898 }
1899 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
1900
ufshcd_clkgate_delay_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1901 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1902 struct device_attribute *attr, const char *buf, size_t count)
1903 {
1904 unsigned long value;
1905
1906 if (kstrtoul(buf, 0, &value))
1907 return -EINVAL;
1908
1909 ufshcd_clkgate_delay_set(dev, value);
1910 return count;
1911 }
1912
ufshcd_clkgate_enable_show(struct device * dev,struct device_attribute * attr,char * buf)1913 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1914 struct device_attribute *attr, char *buf)
1915 {
1916 struct ufs_hba *hba = dev_get_drvdata(dev);
1917
1918 return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
1919 }
1920
ufshcd_clkgate_enable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1921 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1922 struct device_attribute *attr, const char *buf, size_t count)
1923 {
1924 struct ufs_hba *hba = dev_get_drvdata(dev);
1925 unsigned long flags;
1926 u32 value;
1927
1928 if (kstrtou32(buf, 0, &value))
1929 return -EINVAL;
1930
1931 value = !!value;
1932
1933 spin_lock_irqsave(hba->host->host_lock, flags);
1934 if (value == hba->clk_gating.is_enabled)
1935 goto out;
1936
1937 if (value)
1938 __ufshcd_release(hba);
1939 else
1940 hba->clk_gating.active_reqs++;
1941
1942 hba->clk_gating.is_enabled = value;
1943 out:
1944 spin_unlock_irqrestore(hba->host->host_lock, flags);
1945 return count;
1946 }
1947
ufshcd_init_clk_gating_sysfs(struct ufs_hba * hba)1948 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
1949 {
1950 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1951 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1952 sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1953 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1954 hba->clk_gating.delay_attr.attr.mode = 0644;
1955 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1956 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1957
1958 hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1959 hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1960 sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1961 hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1962 hba->clk_gating.enable_attr.attr.mode = 0644;
1963 if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1964 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1965 }
1966
ufshcd_remove_clk_gating_sysfs(struct ufs_hba * hba)1967 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
1968 {
1969 if (hba->clk_gating.delay_attr.attr.name)
1970 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1971 if (hba->clk_gating.enable_attr.attr.name)
1972 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1973 }
1974
ufshcd_init_clk_gating(struct ufs_hba * hba)1975 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1976 {
1977 char wq_name[sizeof("ufs_clk_gating_00")];
1978
1979 if (!ufshcd_is_clkgating_allowed(hba))
1980 return;
1981
1982 hba->clk_gating.state = CLKS_ON;
1983
1984 hba->clk_gating.delay_ms = 150;
1985 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1986 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1987
1988 snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1989 hba->host->host_no);
1990 hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1991 WQ_MEM_RECLAIM | WQ_HIGHPRI);
1992
1993 ufshcd_init_clk_gating_sysfs(hba);
1994
1995 hba->clk_gating.is_enabled = true;
1996 hba->clk_gating.is_initialized = true;
1997 }
1998
ufshcd_exit_clk_gating(struct ufs_hba * hba)1999 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
2000 {
2001 if (!hba->clk_gating.is_initialized)
2002 return;
2003
2004 ufshcd_remove_clk_gating_sysfs(hba);
2005
2006 /* Ungate the clock if necessary. */
2007 ufshcd_hold(hba, false);
2008 hba->clk_gating.is_initialized = false;
2009 ufshcd_release(hba);
2010
2011 destroy_workqueue(hba->clk_gating.clk_gating_workq);
2012 }
2013
2014 /* Must be called with host lock acquired */
ufshcd_clk_scaling_start_busy(struct ufs_hba * hba)2015 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2016 {
2017 bool queue_resume_work = false;
2018 ktime_t curr_t = ktime_get();
2019 unsigned long flags;
2020
2021 if (!ufshcd_is_clkscaling_supported(hba))
2022 return;
2023
2024 spin_lock_irqsave(hba->host->host_lock, flags);
2025 if (!hba->clk_scaling.active_reqs++)
2026 queue_resume_work = true;
2027
2028 if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2029 spin_unlock_irqrestore(hba->host->host_lock, flags);
2030 return;
2031 }
2032
2033 if (queue_resume_work)
2034 queue_work(hba->clk_scaling.workq,
2035 &hba->clk_scaling.resume_work);
2036
2037 if (!hba->clk_scaling.window_start_t) {
2038 hba->clk_scaling.window_start_t = curr_t;
2039 hba->clk_scaling.tot_busy_t = 0;
2040 hba->clk_scaling.is_busy_started = false;
2041 }
2042
2043 if (!hba->clk_scaling.is_busy_started) {
2044 hba->clk_scaling.busy_start_t = curr_t;
2045 hba->clk_scaling.is_busy_started = true;
2046 }
2047 spin_unlock_irqrestore(hba->host->host_lock, flags);
2048 }
2049
ufshcd_clk_scaling_update_busy(struct ufs_hba * hba)2050 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2051 {
2052 struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2053 unsigned long flags;
2054
2055 if (!ufshcd_is_clkscaling_supported(hba))
2056 return;
2057
2058 spin_lock_irqsave(hba->host->host_lock, flags);
2059 hba->clk_scaling.active_reqs--;
2060 if (!hba->outstanding_reqs && scaling->is_busy_started) {
2061 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2062 scaling->busy_start_t));
2063 scaling->busy_start_t = 0;
2064 scaling->is_busy_started = false;
2065 }
2066 spin_unlock_irqrestore(hba->host->host_lock, flags);
2067 }
2068
ufshcd_monitor_opcode2dir(u8 opcode)2069 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2070 {
2071 if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2072 return READ;
2073 else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2074 return WRITE;
2075 else
2076 return -EINVAL;
2077 }
2078
ufshcd_should_inform_monitor(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2079 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2080 struct ufshcd_lrb *lrbp)
2081 {
2082 const struct ufs_hba_monitor *m = &hba->monitor;
2083
2084 return (m->enabled && lrbp && lrbp->cmd &&
2085 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2086 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2087 }
2088
ufshcd_start_monitor(struct ufs_hba * hba,const struct ufshcd_lrb * lrbp)2089 static void ufshcd_start_monitor(struct ufs_hba *hba,
2090 const struct ufshcd_lrb *lrbp)
2091 {
2092 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2093 unsigned long flags;
2094
2095 spin_lock_irqsave(hba->host->host_lock, flags);
2096 if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2097 hba->monitor.busy_start_ts[dir] = ktime_get();
2098 spin_unlock_irqrestore(hba->host->host_lock, flags);
2099 }
2100
ufshcd_update_monitor(struct ufs_hba * hba,const struct ufshcd_lrb * lrbp)2101 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp)
2102 {
2103 int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2104 unsigned long flags;
2105
2106 spin_lock_irqsave(hba->host->host_lock, flags);
2107 if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2108 const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2109 struct ufs_hba_monitor *m = &hba->monitor;
2110 ktime_t now, inc, lat;
2111
2112 now = lrbp->compl_time_stamp;
2113 inc = ktime_sub(now, m->busy_start_ts[dir]);
2114 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2115 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2116
2117 /* Update latencies */
2118 m->nr_req[dir]++;
2119 lat = ktime_sub(now, lrbp->issue_time_stamp);
2120 m->lat_sum[dir] += lat;
2121 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2122 m->lat_max[dir] = lat;
2123 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2124 m->lat_min[dir] = lat;
2125
2126 m->nr_queued[dir]--;
2127 /* Push forward the busy start of monitor */
2128 m->busy_start_ts[dir] = now;
2129 }
2130 spin_unlock_irqrestore(hba->host->host_lock, flags);
2131 }
2132
2133 /**
2134 * ufshcd_send_command - Send SCSI or device management commands
2135 * @hba: per adapter instance
2136 * @task_tag: Task tag of the command
2137 */
2138 static inline
ufshcd_send_command(struct ufs_hba * hba,unsigned int task_tag)2139 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2140 {
2141 struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2142 unsigned long flags;
2143
2144 lrbp->issue_time_stamp = ktime_get();
2145 lrbp->issue_time_stamp_local_clock = local_clock();
2146 lrbp->compl_time_stamp = ktime_set(0, 0);
2147 lrbp->compl_time_stamp_local_clock = 0;
2148 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2149 ufshcd_clk_scaling_start_busy(hba);
2150 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2151 ufshcd_start_monitor(hba, lrbp);
2152
2153 spin_lock_irqsave(&hba->outstanding_lock, flags);
2154 if (hba->vops && hba->vops->setup_xfer_req)
2155 hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd);
2156 __set_bit(task_tag, &hba->outstanding_reqs);
2157 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
2158 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2159 }
2160
2161 /**
2162 * ufshcd_copy_sense_data - Copy sense data in case of check condition
2163 * @lrbp: pointer to local reference block
2164 */
ufshcd_copy_sense_data(struct ufshcd_lrb * lrbp)2165 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2166 {
2167 u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2168 int len;
2169
2170 if (sense_buffer &&
2171 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
2172 int len_to_copy;
2173
2174 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2175 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2176
2177 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2178 len_to_copy);
2179 }
2180 }
2181
2182 /**
2183 * ufshcd_copy_query_response() - Copy the Query Response and the data
2184 * descriptor
2185 * @hba: per adapter instance
2186 * @lrbp: pointer to local reference block
2187 */
2188 static
ufshcd_copy_query_response(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2189 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2190 {
2191 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2192
2193 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2194
2195 /* Get the descriptor */
2196 if (hba->dev_cmd.query.descriptor &&
2197 lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2198 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2199 GENERAL_UPIU_REQUEST_SIZE;
2200 u16 resp_len;
2201 u16 buf_len;
2202
2203 /* data segment length */
2204 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
2205 MASK_QUERY_DATA_SEG_LEN;
2206 buf_len = be16_to_cpu(
2207 hba->dev_cmd.query.request.upiu_req.length);
2208 if (likely(buf_len >= resp_len)) {
2209 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2210 } else {
2211 dev_warn(hba->dev,
2212 "%s: rsp size %d is bigger than buffer size %d",
2213 __func__, resp_len, buf_len);
2214 return -EINVAL;
2215 }
2216 }
2217
2218 return 0;
2219 }
2220
2221 /**
2222 * ufshcd_hba_capabilities - Read controller capabilities
2223 * @hba: per adapter instance
2224 *
2225 * Return: 0 on success, negative on error.
2226 */
ufshcd_hba_capabilities(struct ufs_hba * hba)2227 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2228 {
2229 int err;
2230
2231 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2232 if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS)
2233 hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT;
2234
2235 /* nutrs and nutmrs are 0 based values */
2236 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2237 hba->nutmrs =
2238 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2239 hba->reserved_slot = hba->nutrs - 1;
2240
2241 /* Read crypto capabilities */
2242 err = ufshcd_hba_init_crypto_capabilities(hba);
2243 if (err)
2244 dev_err(hba->dev, "crypto setup failed\n");
2245
2246 return err;
2247 }
2248
2249 /**
2250 * ufshcd_ready_for_uic_cmd - Check if controller is ready
2251 * to accept UIC commands
2252 * @hba: per adapter instance
2253 * Return true on success, else false
2254 */
ufshcd_ready_for_uic_cmd(struct ufs_hba * hba)2255 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2256 {
2257 return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY;
2258 }
2259
2260 /**
2261 * ufshcd_get_upmcrs - Get the power mode change request status
2262 * @hba: Pointer to adapter instance
2263 *
2264 * This function gets the UPMCRS field of HCS register
2265 * Returns value of UPMCRS field
2266 */
ufshcd_get_upmcrs(struct ufs_hba * hba)2267 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2268 {
2269 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2270 }
2271
2272 /**
2273 * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2274 * @hba: per adapter instance
2275 * @uic_cmd: UIC command
2276 */
2277 static inline void
ufshcd_dispatch_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2278 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2279 {
2280 lockdep_assert_held(&hba->uic_cmd_mutex);
2281
2282 WARN_ON(hba->active_uic_cmd);
2283
2284 hba->active_uic_cmd = uic_cmd;
2285
2286 /* Write Args */
2287 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2288 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2289 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2290
2291 ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2292
2293 /* Write UIC Cmd */
2294 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2295 REG_UIC_COMMAND);
2296 }
2297
2298 /**
2299 * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2300 * @hba: per adapter instance
2301 * @uic_cmd: UIC command
2302 *
2303 * Returns 0 only if success.
2304 */
2305 static int
ufshcd_wait_for_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2306 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2307 {
2308 int ret;
2309 unsigned long flags;
2310
2311 lockdep_assert_held(&hba->uic_cmd_mutex);
2312
2313 if (wait_for_completion_timeout(&uic_cmd->done,
2314 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2315 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2316 } else {
2317 ret = -ETIMEDOUT;
2318 dev_err(hba->dev,
2319 "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2320 uic_cmd->command, uic_cmd->argument3);
2321
2322 if (!uic_cmd->cmd_active) {
2323 dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2324 __func__);
2325 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2326 }
2327 }
2328
2329 spin_lock_irqsave(hba->host->host_lock, flags);
2330 hba->active_uic_cmd = NULL;
2331 spin_unlock_irqrestore(hba->host->host_lock, flags);
2332
2333 return ret;
2334 }
2335
2336 /**
2337 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2338 * @hba: per adapter instance
2339 * @uic_cmd: UIC command
2340 * @completion: initialize the completion only if this is set to true
2341 *
2342 * Returns 0 only if success.
2343 */
2344 static int
__ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd,bool completion)2345 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2346 bool completion)
2347 {
2348 lockdep_assert_held(&hba->uic_cmd_mutex);
2349 lockdep_assert_held(hba->host->host_lock);
2350
2351 if (!ufshcd_ready_for_uic_cmd(hba)) {
2352 dev_err(hba->dev,
2353 "Controller not ready to accept UIC commands\n");
2354 return -EIO;
2355 }
2356
2357 if (completion)
2358 init_completion(&uic_cmd->done);
2359
2360 uic_cmd->cmd_active = 1;
2361 ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2362
2363 return 0;
2364 }
2365
2366 /**
2367 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2368 * @hba: per adapter instance
2369 * @uic_cmd: UIC command
2370 *
2371 * Returns 0 only if success.
2372 */
ufshcd_send_uic_cmd(struct ufs_hba * hba,struct uic_command * uic_cmd)2373 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2374 {
2375 int ret;
2376 unsigned long flags;
2377
2378 if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2379 return 0;
2380
2381 ufshcd_hold(hba, false);
2382 mutex_lock(&hba->uic_cmd_mutex);
2383 ufshcd_add_delay_before_dme_cmd(hba);
2384
2385 spin_lock_irqsave(hba->host->host_lock, flags);
2386 ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2387 spin_unlock_irqrestore(hba->host->host_lock, flags);
2388 if (!ret)
2389 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2390
2391 mutex_unlock(&hba->uic_cmd_mutex);
2392
2393 ufshcd_release(hba);
2394 return ret;
2395 }
2396
2397 /**
2398 * ufshcd_map_sg - Map scatter-gather list to prdt
2399 * @hba: per adapter instance
2400 * @lrbp: pointer to local reference block
2401 *
2402 * Returns 0 in case of success, non-zero value in case of failure
2403 */
ufshcd_map_sg(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2404 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2405 {
2406 struct ufshcd_sg_entry *prd_table;
2407 struct scatterlist *sg;
2408 struct scsi_cmnd *cmd;
2409 int sg_segments;
2410 int i;
2411
2412 cmd = lrbp->cmd;
2413 sg_segments = scsi_dma_map(cmd);
2414 if (sg_segments < 0)
2415 return sg_segments;
2416
2417 if (sg_segments) {
2418
2419 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2420 lrbp->utr_descriptor_ptr->prd_table_length =
2421 cpu_to_le16((sg_segments *
2422 sizeof(struct ufshcd_sg_entry)));
2423 else
2424 lrbp->utr_descriptor_ptr->prd_table_length =
2425 cpu_to_le16(sg_segments);
2426
2427 prd_table = lrbp->ucd_prdt_ptr;
2428
2429 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2430 const unsigned int len = sg_dma_len(sg);
2431
2432 /*
2433 * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2434 * based value that indicates the length, in bytes, of
2435 * the data block. A maximum of length of 256KB may
2436 * exist for any entry. Bits 1:0 of this field shall be
2437 * 11b to indicate Dword granularity. A value of '3'
2438 * indicates 4 bytes, '7' indicates 8 bytes, etc."
2439 */
2440 WARN_ONCE(len > 256 * 1024, "len = %#x\n", len);
2441 prd_table[i].size = cpu_to_le32(len - 1);
2442 prd_table[i].addr = cpu_to_le64(sg->dma_address);
2443 prd_table[i].reserved = 0;
2444 }
2445 } else {
2446 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2447 }
2448
2449 return 0;
2450 }
2451
2452 /**
2453 * ufshcd_enable_intr - enable interrupts
2454 * @hba: per adapter instance
2455 * @intrs: interrupt bits
2456 */
ufshcd_enable_intr(struct ufs_hba * hba,u32 intrs)2457 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2458 {
2459 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2460
2461 if (hba->ufs_version == ufshci_version(1, 0)) {
2462 u32 rw;
2463 rw = set & INTERRUPT_MASK_RW_VER_10;
2464 set = rw | ((set ^ intrs) & intrs);
2465 } else {
2466 set |= intrs;
2467 }
2468
2469 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2470 }
2471
2472 /**
2473 * ufshcd_disable_intr - disable interrupts
2474 * @hba: per adapter instance
2475 * @intrs: interrupt bits
2476 */
ufshcd_disable_intr(struct ufs_hba * hba,u32 intrs)2477 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2478 {
2479 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2480
2481 if (hba->ufs_version == ufshci_version(1, 0)) {
2482 u32 rw;
2483 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2484 ~(intrs & INTERRUPT_MASK_RW_VER_10);
2485 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2486
2487 } else {
2488 set &= ~intrs;
2489 }
2490
2491 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2492 }
2493
2494 /**
2495 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2496 * descriptor according to request
2497 * @lrbp: pointer to local reference block
2498 * @upiu_flags: flags required in the header
2499 * @cmd_dir: requests data direction
2500 */
ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb * lrbp,u8 * upiu_flags,enum dma_data_direction cmd_dir)2501 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2502 u8 *upiu_flags, enum dma_data_direction cmd_dir)
2503 {
2504 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2505 u32 data_direction;
2506 u32 dword_0;
2507 u32 dword_1 = 0;
2508 u32 dword_3 = 0;
2509
2510 if (cmd_dir == DMA_FROM_DEVICE) {
2511 data_direction = UTP_DEVICE_TO_HOST;
2512 *upiu_flags = UPIU_CMD_FLAGS_READ;
2513 } else if (cmd_dir == DMA_TO_DEVICE) {
2514 data_direction = UTP_HOST_TO_DEVICE;
2515 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2516 } else {
2517 data_direction = UTP_NO_DATA_TRANSFER;
2518 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2519 }
2520
2521 dword_0 = data_direction | (lrbp->command_type
2522 << UPIU_COMMAND_TYPE_OFFSET);
2523 if (lrbp->intr_cmd)
2524 dword_0 |= UTP_REQ_DESC_INT_CMD;
2525
2526 /* Prepare crypto related dwords */
2527 ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2528
2529 /* Transfer request descriptor header fields */
2530 req_desc->header.dword_0 = cpu_to_le32(dword_0);
2531 req_desc->header.dword_1 = cpu_to_le32(dword_1);
2532 /*
2533 * assigning invalid value for command status. Controller
2534 * updates OCS on command completion, with the command
2535 * status
2536 */
2537 req_desc->header.dword_2 =
2538 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2539 req_desc->header.dword_3 = cpu_to_le32(dword_3);
2540
2541 req_desc->prd_table_length = 0;
2542 }
2543
2544 /**
2545 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2546 * for scsi commands
2547 * @lrbp: local reference block pointer
2548 * @upiu_flags: flags
2549 */
2550 static
ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb * lrbp,u8 upiu_flags)2551 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2552 {
2553 struct scsi_cmnd *cmd = lrbp->cmd;
2554 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2555 unsigned short cdb_len;
2556
2557 /* command descriptor fields */
2558 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2559 UPIU_TRANSACTION_COMMAND, upiu_flags,
2560 lrbp->lun, lrbp->task_tag);
2561 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2562 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2563
2564 /* Total EHS length and Data segment length will be zero */
2565 ucd_req_ptr->header.dword_2 = 0;
2566
2567 ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2568
2569 cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2570 memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2571 memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2572
2573 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2574 }
2575
2576 /**
2577 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2578 * for query requsts
2579 * @hba: UFS hba
2580 * @lrbp: local reference block pointer
2581 * @upiu_flags: flags
2582 */
ufshcd_prepare_utp_query_req_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,u8 upiu_flags)2583 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2584 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2585 {
2586 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2587 struct ufs_query *query = &hba->dev_cmd.query;
2588 u16 len = be16_to_cpu(query->request.upiu_req.length);
2589
2590 /* Query request header */
2591 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2592 UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2593 lrbp->lun, lrbp->task_tag);
2594 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2595 0, query->request.query_func, 0, 0);
2596
2597 /* Data segment length only need for WRITE_DESC */
2598 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2599 ucd_req_ptr->header.dword_2 =
2600 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2601 else
2602 ucd_req_ptr->header.dword_2 = 0;
2603
2604 /* Copy the Query Request buffer as is */
2605 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2606 QUERY_OSF_SIZE);
2607
2608 /* Copy the Descriptor */
2609 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2610 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2611
2612 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2613 }
2614
ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb * lrbp)2615 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2616 {
2617 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2618
2619 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2620
2621 /* command descriptor fields */
2622 ucd_req_ptr->header.dword_0 =
2623 UPIU_HEADER_DWORD(
2624 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2625 /* clear rest of the fields of basic header */
2626 ucd_req_ptr->header.dword_1 = 0;
2627 ucd_req_ptr->header.dword_2 = 0;
2628
2629 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2630 }
2631
2632 /**
2633 * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2634 * for Device Management Purposes
2635 * @hba: per adapter instance
2636 * @lrbp: pointer to local reference block
2637 */
ufshcd_compose_devman_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2638 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2639 struct ufshcd_lrb *lrbp)
2640 {
2641 u8 upiu_flags;
2642 int ret = 0;
2643
2644 if (hba->ufs_version <= ufshci_version(1, 1))
2645 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2646 else
2647 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2648
2649 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2650 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2651 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2652 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2653 ufshcd_prepare_utp_nop_upiu(lrbp);
2654 else
2655 ret = -EINVAL;
2656
2657 return ret;
2658 }
2659
2660 /**
2661 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2662 * for SCSI Purposes
2663 * @hba: per adapter instance
2664 * @lrbp: pointer to local reference block
2665 */
ufshcd_comp_scsi_upiu(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2666 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2667 {
2668 u8 upiu_flags;
2669 int ret = 0;
2670
2671 if (hba->ufs_version <= ufshci_version(1, 1))
2672 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2673 else
2674 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2675
2676 if (likely(lrbp->cmd)) {
2677 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2678 lrbp->cmd->sc_data_direction);
2679 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2680 } else {
2681 ret = -EINVAL;
2682 }
2683
2684 return ret;
2685 }
2686
2687 /**
2688 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2689 * @upiu_wlun_id: UPIU W-LUN id
2690 *
2691 * Returns SCSI W-LUN id
2692 */
ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)2693 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2694 {
2695 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2696 }
2697
is_device_wlun(struct scsi_device * sdev)2698 static inline bool is_device_wlun(struct scsi_device *sdev)
2699 {
2700 return sdev->lun ==
2701 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2702 }
2703
2704 /*
2705 * Associate the UFS controller queue with the default and poll HCTX types.
2706 * Initialize the mq_map[] arrays.
2707 */
ufshcd_map_queues(struct Scsi_Host * shost)2708 static void ufshcd_map_queues(struct Scsi_Host *shost)
2709 {
2710 int i;
2711
2712 for (i = 0; i < shost->nr_maps; i++) {
2713 struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2714
2715 switch (i) {
2716 case HCTX_TYPE_DEFAULT:
2717 case HCTX_TYPE_POLL:
2718 map->nr_queues = 1;
2719 break;
2720 case HCTX_TYPE_READ:
2721 map->nr_queues = 0;
2722 continue;
2723 default:
2724 WARN_ON_ONCE(true);
2725 }
2726 map->queue_offset = 0;
2727 blk_mq_map_queues(map);
2728 }
2729 }
2730
ufshcd_init_lrb(struct ufs_hba * hba,struct ufshcd_lrb * lrb,int i)2731 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2732 {
2733 struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2734 struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2735 dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2736 i * sizeof(struct utp_transfer_cmd_desc);
2737 u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2738 response_upiu);
2739 u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2740
2741 lrb->utr_descriptor_ptr = utrdlp + i;
2742 lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2743 i * sizeof(struct utp_transfer_req_desc);
2744 lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2745 lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2746 lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2747 lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2748 lrb->ucd_prdt_ptr = cmd_descp[i].prd_table;
2749 lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2750 }
2751
2752 /**
2753 * ufshcd_queuecommand - main entry point for SCSI requests
2754 * @host: SCSI host pointer
2755 * @cmd: command from SCSI Midlayer
2756 *
2757 * Returns 0 for success, non-zero in case of failure
2758 */
ufshcd_queuecommand(struct Scsi_Host * host,struct scsi_cmnd * cmd)2759 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2760 {
2761 struct ufs_hba *hba = shost_priv(host);
2762 int tag = scsi_cmd_to_rq(cmd)->tag;
2763 struct ufshcd_lrb *lrbp;
2764 int err = 0;
2765
2766 WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag);
2767
2768 /*
2769 * Allows the UFS error handler to wait for prior ufshcd_queuecommand()
2770 * calls.
2771 */
2772 rcu_read_lock();
2773
2774 switch (hba->ufshcd_state) {
2775 case UFSHCD_STATE_OPERATIONAL:
2776 break;
2777 case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2778 /*
2779 * SCSI error handler can call ->queuecommand() while UFS error
2780 * handler is in progress. Error interrupts could change the
2781 * state from UFSHCD_STATE_RESET to
2782 * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2783 * being issued in that case.
2784 */
2785 if (ufshcd_eh_in_progress(hba)) {
2786 err = SCSI_MLQUEUE_HOST_BUSY;
2787 goto out;
2788 }
2789 break;
2790 case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2791 /*
2792 * pm_runtime_get_sync() is used at error handling preparation
2793 * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2794 * PM ops, it can never be finished if we let SCSI layer keep
2795 * retrying it, which gets err handler stuck forever. Neither
2796 * can we let the scsi cmd pass through, because UFS is in bad
2797 * state, the scsi cmd may eventually time out, which will get
2798 * err handler blocked for too long. So, just fail the scsi cmd
2799 * sent from PM ops, err handler can recover PM error anyways.
2800 */
2801 if (hba->pm_op_in_progress) {
2802 hba->force_reset = true;
2803 set_host_byte(cmd, DID_BAD_TARGET);
2804 scsi_done(cmd);
2805 goto out;
2806 }
2807 fallthrough;
2808 case UFSHCD_STATE_RESET:
2809 err = SCSI_MLQUEUE_HOST_BUSY;
2810 goto out;
2811 case UFSHCD_STATE_ERROR:
2812 set_host_byte(cmd, DID_ERROR);
2813 scsi_done(cmd);
2814 goto out;
2815 }
2816
2817 hba->req_abort_count = 0;
2818
2819 err = ufshcd_hold(hba, true);
2820 if (err) {
2821 err = SCSI_MLQUEUE_HOST_BUSY;
2822 goto out;
2823 }
2824 WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2825 (hba->clk_gating.state != CLKS_ON));
2826
2827 lrbp = &hba->lrb[tag];
2828 WARN_ON(lrbp->cmd);
2829 lrbp->cmd = cmd;
2830 lrbp->task_tag = tag;
2831 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2832 lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
2833
2834 ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp);
2835
2836 lrbp->req_abort_skip = false;
2837
2838 ufshpb_prep(hba, lrbp);
2839
2840 ufshcd_comp_scsi_upiu(hba, lrbp);
2841
2842 err = ufshcd_map_sg(hba, lrbp);
2843 if (err) {
2844 lrbp->cmd = NULL;
2845 ufshcd_release(hba);
2846 goto out;
2847 }
2848
2849 ufshcd_send_command(hba, tag);
2850
2851 out:
2852 rcu_read_unlock();
2853
2854 if (ufs_trigger_eh()) {
2855 unsigned long flags;
2856
2857 spin_lock_irqsave(hba->host->host_lock, flags);
2858 ufshcd_schedule_eh_work(hba);
2859 spin_unlock_irqrestore(hba->host->host_lock, flags);
2860 }
2861
2862 return err;
2863 }
2864
ufshcd_compose_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,enum dev_cmd_type cmd_type,int tag)2865 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2866 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2867 {
2868 lrbp->cmd = NULL;
2869 lrbp->task_tag = tag;
2870 lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2871 lrbp->intr_cmd = true; /* No interrupt aggregation */
2872 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
2873 hba->dev_cmd.type = cmd_type;
2874
2875 return ufshcd_compose_devman_upiu(hba, lrbp);
2876 }
2877
2878 /*
2879 * Clear all the requests from the controller for which a bit has been set in
2880 * @mask and wait until the controller confirms that these requests have been
2881 * cleared.
2882 */
ufshcd_clear_cmds(struct ufs_hba * hba,u32 mask)2883 static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask)
2884 {
2885 unsigned long flags;
2886
2887 /* clear outstanding transaction before retry */
2888 spin_lock_irqsave(hba->host->host_lock, flags);
2889 ufshcd_utrl_clear(hba, mask);
2890 spin_unlock_irqrestore(hba->host->host_lock, flags);
2891
2892 /*
2893 * wait for h/w to clear corresponding bit in door-bell.
2894 * max. wait is 1 sec.
2895 */
2896 return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL,
2897 mask, ~mask, 1000, 1000);
2898 }
2899
2900 static int
ufshcd_check_query_response(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2901 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2902 {
2903 struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2904
2905 /* Get the UPIU response */
2906 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2907 UPIU_RSP_CODE_OFFSET;
2908 return query_res->response;
2909 }
2910
2911 /**
2912 * ufshcd_dev_cmd_completion() - handles device management command responses
2913 * @hba: per adapter instance
2914 * @lrbp: pointer to local reference block
2915 */
2916 static int
ufshcd_dev_cmd_completion(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)2917 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2918 {
2919 int resp;
2920 int err = 0;
2921
2922 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2923 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2924
2925 switch (resp) {
2926 case UPIU_TRANSACTION_NOP_IN:
2927 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2928 err = -EINVAL;
2929 dev_err(hba->dev, "%s: unexpected response %x\n",
2930 __func__, resp);
2931 }
2932 break;
2933 case UPIU_TRANSACTION_QUERY_RSP:
2934 err = ufshcd_check_query_response(hba, lrbp);
2935 if (!err)
2936 err = ufshcd_copy_query_response(hba, lrbp);
2937 break;
2938 case UPIU_TRANSACTION_REJECT_UPIU:
2939 /* TODO: handle Reject UPIU Response */
2940 err = -EPERM;
2941 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2942 __func__);
2943 break;
2944 default:
2945 err = -EINVAL;
2946 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2947 __func__, resp);
2948 break;
2949 }
2950
2951 return err;
2952 }
2953
ufshcd_wait_for_dev_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp,int max_timeout)2954 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2955 struct ufshcd_lrb *lrbp, int max_timeout)
2956 {
2957 unsigned long time_left = msecs_to_jiffies(max_timeout);
2958 unsigned long flags;
2959 bool pending;
2960 int err;
2961
2962 retry:
2963 time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2964 time_left);
2965
2966 if (likely(time_left)) {
2967 /*
2968 * The completion handler called complete() and the caller of
2969 * this function still owns the @lrbp tag so the code below does
2970 * not trigger any race conditions.
2971 */
2972 hba->dev_cmd.complete = NULL;
2973 err = ufshcd_get_tr_ocs(lrbp);
2974 if (!err)
2975 err = ufshcd_dev_cmd_completion(hba, lrbp);
2976 } else {
2977 err = -ETIMEDOUT;
2978 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2979 __func__, lrbp->task_tag);
2980 if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) {
2981 /* successfully cleared the command, retry if needed */
2982 err = -EAGAIN;
2983 /*
2984 * Since clearing the command succeeded we also need to
2985 * clear the task tag bit from the outstanding_reqs
2986 * variable.
2987 */
2988 spin_lock_irqsave(&hba->outstanding_lock, flags);
2989 pending = test_bit(lrbp->task_tag,
2990 &hba->outstanding_reqs);
2991 if (pending) {
2992 hba->dev_cmd.complete = NULL;
2993 __clear_bit(lrbp->task_tag,
2994 &hba->outstanding_reqs);
2995 }
2996 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2997
2998 if (!pending) {
2999 /*
3000 * The completion handler ran while we tried to
3001 * clear the command.
3002 */
3003 time_left = 1;
3004 goto retry;
3005 }
3006 } else {
3007 dev_err(hba->dev, "%s: failed to clear tag %d\n",
3008 __func__, lrbp->task_tag);
3009 }
3010 }
3011
3012 return err;
3013 }
3014
3015 /**
3016 * ufshcd_exec_dev_cmd - API for sending device management requests
3017 * @hba: UFS hba
3018 * @cmd_type: specifies the type (NOP, Query...)
3019 * @timeout: timeout in milliseconds
3020 *
3021 * NOTE: Since there is only one available tag for device management commands,
3022 * it is expected you hold the hba->dev_cmd.lock mutex.
3023 */
ufshcd_exec_dev_cmd(struct ufs_hba * hba,enum dev_cmd_type cmd_type,int timeout)3024 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
3025 enum dev_cmd_type cmd_type, int timeout)
3026 {
3027 DECLARE_COMPLETION_ONSTACK(wait);
3028 const u32 tag = hba->reserved_slot;
3029 struct ufshcd_lrb *lrbp;
3030 int err;
3031
3032 /* Protects use of hba->reserved_slot. */
3033 lockdep_assert_held(&hba->dev_cmd.lock);
3034
3035 down_read(&hba->clk_scaling_lock);
3036
3037 lrbp = &hba->lrb[tag];
3038 WARN_ON(lrbp->cmd);
3039 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3040 if (unlikely(err))
3041 goto out;
3042
3043 hba->dev_cmd.complete = &wait;
3044
3045 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3046
3047 ufshcd_send_command(hba, tag);
3048 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
3049 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3050 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3051
3052 out:
3053 up_read(&hba->clk_scaling_lock);
3054 return err;
3055 }
3056
3057 /**
3058 * ufshcd_init_query() - init the query response and request parameters
3059 * @hba: per-adapter instance
3060 * @request: address of the request pointer to be initialized
3061 * @response: address of the response pointer to be initialized
3062 * @opcode: operation to perform
3063 * @idn: flag idn to access
3064 * @index: LU number to access
3065 * @selector: query/flag/descriptor further identification
3066 */
ufshcd_init_query(struct ufs_hba * hba,struct ufs_query_req ** request,struct ufs_query_res ** response,enum query_opcode opcode,u8 idn,u8 index,u8 selector)3067 static inline void ufshcd_init_query(struct ufs_hba *hba,
3068 struct ufs_query_req **request, struct ufs_query_res **response,
3069 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3070 {
3071 *request = &hba->dev_cmd.query.request;
3072 *response = &hba->dev_cmd.query.response;
3073 memset(*request, 0, sizeof(struct ufs_query_req));
3074 memset(*response, 0, sizeof(struct ufs_query_res));
3075 (*request)->upiu_req.opcode = opcode;
3076 (*request)->upiu_req.idn = idn;
3077 (*request)->upiu_req.index = index;
3078 (*request)->upiu_req.selector = selector;
3079 }
3080
ufshcd_query_flag_retry(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3081 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3082 enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3083 {
3084 int ret;
3085 int retries;
3086
3087 for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3088 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3089 if (ret)
3090 dev_dbg(hba->dev,
3091 "%s: failed with error %d, retries %d\n",
3092 __func__, ret, retries);
3093 else
3094 break;
3095 }
3096
3097 if (ret)
3098 dev_err(hba->dev,
3099 "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n",
3100 __func__, opcode, idn, ret, retries);
3101 return ret;
3102 }
3103
3104 /**
3105 * ufshcd_query_flag() - API function for sending flag query requests
3106 * @hba: per-adapter instance
3107 * @opcode: flag query to perform
3108 * @idn: flag idn to access
3109 * @index: flag index to access
3110 * @flag_res: the flag value after the query request completes
3111 *
3112 * Returns 0 for success, non-zero in case of failure
3113 */
ufshcd_query_flag(struct ufs_hba * hba,enum query_opcode opcode,enum flag_idn idn,u8 index,bool * flag_res)3114 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3115 enum flag_idn idn, u8 index, bool *flag_res)
3116 {
3117 struct ufs_query_req *request = NULL;
3118 struct ufs_query_res *response = NULL;
3119 int err, selector = 0;
3120 int timeout = QUERY_REQ_TIMEOUT;
3121
3122 BUG_ON(!hba);
3123
3124 ufshcd_hold(hba, false);
3125 mutex_lock(&hba->dev_cmd.lock);
3126 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3127 selector);
3128
3129 switch (opcode) {
3130 case UPIU_QUERY_OPCODE_SET_FLAG:
3131 case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3132 case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3133 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3134 break;
3135 case UPIU_QUERY_OPCODE_READ_FLAG:
3136 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3137 if (!flag_res) {
3138 /* No dummy reads */
3139 dev_err(hba->dev, "%s: Invalid argument for read request\n",
3140 __func__);
3141 err = -EINVAL;
3142 goto out_unlock;
3143 }
3144 break;
3145 default:
3146 dev_err(hba->dev,
3147 "%s: Expected query flag opcode but got = %d\n",
3148 __func__, opcode);
3149 err = -EINVAL;
3150 goto out_unlock;
3151 }
3152
3153 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3154
3155 if (err) {
3156 dev_err(hba->dev,
3157 "%s: Sending flag query for idn %d failed, err = %d\n",
3158 __func__, idn, err);
3159 goto out_unlock;
3160 }
3161
3162 if (flag_res)
3163 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3164 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3165
3166 out_unlock:
3167 mutex_unlock(&hba->dev_cmd.lock);
3168 ufshcd_release(hba);
3169 return err;
3170 }
3171
3172 /**
3173 * ufshcd_query_attr - API function for sending attribute requests
3174 * @hba: per-adapter instance
3175 * @opcode: attribute opcode
3176 * @idn: attribute idn to access
3177 * @index: index field
3178 * @selector: selector field
3179 * @attr_val: the attribute value after the query request completes
3180 *
3181 * Returns 0 for success, non-zero in case of failure
3182 */
ufshcd_query_attr(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3183 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3184 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3185 {
3186 struct ufs_query_req *request = NULL;
3187 struct ufs_query_res *response = NULL;
3188 int err;
3189
3190 BUG_ON(!hba);
3191
3192 if (!attr_val) {
3193 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3194 __func__, opcode);
3195 return -EINVAL;
3196 }
3197
3198 ufshcd_hold(hba, false);
3199
3200 mutex_lock(&hba->dev_cmd.lock);
3201 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3202 selector);
3203
3204 switch (opcode) {
3205 case UPIU_QUERY_OPCODE_WRITE_ATTR:
3206 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3207 request->upiu_req.value = cpu_to_be32(*attr_val);
3208 break;
3209 case UPIU_QUERY_OPCODE_READ_ATTR:
3210 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3211 break;
3212 default:
3213 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3214 __func__, opcode);
3215 err = -EINVAL;
3216 goto out_unlock;
3217 }
3218
3219 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3220
3221 if (err) {
3222 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3223 __func__, opcode, idn, index, err);
3224 goto out_unlock;
3225 }
3226
3227 *attr_val = be32_to_cpu(response->upiu_res.value);
3228
3229 out_unlock:
3230 mutex_unlock(&hba->dev_cmd.lock);
3231 ufshcd_release(hba);
3232 return err;
3233 }
3234
3235 /**
3236 * ufshcd_query_attr_retry() - API function for sending query
3237 * attribute with retries
3238 * @hba: per-adapter instance
3239 * @opcode: attribute opcode
3240 * @idn: attribute idn to access
3241 * @index: index field
3242 * @selector: selector field
3243 * @attr_val: the attribute value after the query request
3244 * completes
3245 *
3246 * Returns 0 for success, non-zero in case of failure
3247 */
ufshcd_query_attr_retry(struct ufs_hba * hba,enum query_opcode opcode,enum attr_idn idn,u8 index,u8 selector,u32 * attr_val)3248 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3249 enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3250 u32 *attr_val)
3251 {
3252 int ret = 0;
3253 u32 retries;
3254
3255 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3256 ret = ufshcd_query_attr(hba, opcode, idn, index,
3257 selector, attr_val);
3258 if (ret)
3259 dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3260 __func__, ret, retries);
3261 else
3262 break;
3263 }
3264
3265 if (ret)
3266 dev_err(hba->dev,
3267 "%s: query attribute, idn %d, failed with error %d after %d retries\n",
3268 __func__, idn, ret, QUERY_REQ_RETRIES);
3269 return ret;
3270 }
3271
__ufshcd_query_descriptor(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3272 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3273 enum query_opcode opcode, enum desc_idn idn, u8 index,
3274 u8 selector, u8 *desc_buf, int *buf_len)
3275 {
3276 struct ufs_query_req *request = NULL;
3277 struct ufs_query_res *response = NULL;
3278 int err;
3279
3280 BUG_ON(!hba);
3281
3282 if (!desc_buf) {
3283 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3284 __func__, opcode);
3285 return -EINVAL;
3286 }
3287
3288 if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3289 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3290 __func__, *buf_len);
3291 return -EINVAL;
3292 }
3293
3294 ufshcd_hold(hba, false);
3295
3296 mutex_lock(&hba->dev_cmd.lock);
3297 ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3298 selector);
3299 hba->dev_cmd.query.descriptor = desc_buf;
3300 request->upiu_req.length = cpu_to_be16(*buf_len);
3301
3302 switch (opcode) {
3303 case UPIU_QUERY_OPCODE_WRITE_DESC:
3304 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3305 break;
3306 case UPIU_QUERY_OPCODE_READ_DESC:
3307 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3308 break;
3309 default:
3310 dev_err(hba->dev,
3311 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3312 __func__, opcode);
3313 err = -EINVAL;
3314 goto out_unlock;
3315 }
3316
3317 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3318
3319 if (err) {
3320 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3321 __func__, opcode, idn, index, err);
3322 goto out_unlock;
3323 }
3324
3325 *buf_len = be16_to_cpu(response->upiu_res.length);
3326
3327 out_unlock:
3328 hba->dev_cmd.query.descriptor = NULL;
3329 mutex_unlock(&hba->dev_cmd.lock);
3330 ufshcd_release(hba);
3331 return err;
3332 }
3333
3334 /**
3335 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3336 * @hba: per-adapter instance
3337 * @opcode: attribute opcode
3338 * @idn: attribute idn to access
3339 * @index: index field
3340 * @selector: selector field
3341 * @desc_buf: the buffer that contains the descriptor
3342 * @buf_len: length parameter passed to the device
3343 *
3344 * Returns 0 for success, non-zero in case of failure.
3345 * The buf_len parameter will contain, on return, the length parameter
3346 * received on the response.
3347 */
ufshcd_query_descriptor_retry(struct ufs_hba * hba,enum query_opcode opcode,enum desc_idn idn,u8 index,u8 selector,u8 * desc_buf,int * buf_len)3348 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3349 enum query_opcode opcode,
3350 enum desc_idn idn, u8 index,
3351 u8 selector,
3352 u8 *desc_buf, int *buf_len)
3353 {
3354 int err;
3355 int retries;
3356
3357 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3358 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3359 selector, desc_buf, buf_len);
3360 if (!err || err == -EINVAL)
3361 break;
3362 }
3363
3364 return err;
3365 }
3366
3367 /**
3368 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3369 * @hba: Pointer to adapter instance
3370 * @desc_id: descriptor idn value
3371 * @desc_len: mapped desc length (out)
3372 */
ufshcd_map_desc_id_to_length(struct ufs_hba * hba,enum desc_idn desc_id,int * desc_len)3373 void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3374 int *desc_len)
3375 {
3376 if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3377 desc_id == QUERY_DESC_IDN_RFU_1)
3378 *desc_len = 0;
3379 else
3380 *desc_len = hba->desc_size[desc_id];
3381 }
3382 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3383
ufshcd_update_desc_length(struct ufs_hba * hba,enum desc_idn desc_id,int desc_index,unsigned char desc_len)3384 static void ufshcd_update_desc_length(struct ufs_hba *hba,
3385 enum desc_idn desc_id, int desc_index,
3386 unsigned char desc_len)
3387 {
3388 if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
3389 desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3390 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3391 * than the RPMB unit, however, both descriptors share the same
3392 * desc_idn, to cover both unit descriptors with one length, we
3393 * choose the normal unit descriptor length by desc_index.
3394 */
3395 hba->desc_size[desc_id] = desc_len;
3396 }
3397
3398 /**
3399 * ufshcd_read_desc_param - read the specified descriptor parameter
3400 * @hba: Pointer to adapter instance
3401 * @desc_id: descriptor idn value
3402 * @desc_index: descriptor index
3403 * @param_offset: offset of the parameter to read
3404 * @param_read_buf: pointer to buffer where parameter would be read
3405 * @param_size: sizeof(param_read_buf)
3406 *
3407 * Return 0 in case of success, non-zero otherwise
3408 */
ufshcd_read_desc_param(struct ufs_hba * hba,enum desc_idn desc_id,int desc_index,u8 param_offset,u8 * param_read_buf,u8 param_size)3409 int ufshcd_read_desc_param(struct ufs_hba *hba,
3410 enum desc_idn desc_id,
3411 int desc_index,
3412 u8 param_offset,
3413 u8 *param_read_buf,
3414 u8 param_size)
3415 {
3416 int ret;
3417 u8 *desc_buf;
3418 int buff_len;
3419 bool is_kmalloc = true;
3420
3421 /* Safety check */
3422 if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3423 return -EINVAL;
3424
3425 /* Get the length of descriptor */
3426 ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3427 if (!buff_len) {
3428 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3429 return -EINVAL;
3430 }
3431
3432 if (param_offset >= buff_len) {
3433 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3434 __func__, param_offset, desc_id, buff_len);
3435 return -EINVAL;
3436 }
3437
3438 /* Check whether we need temp memory */
3439 if (param_offset != 0 || param_size < buff_len) {
3440 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3441 if (!desc_buf)
3442 return -ENOMEM;
3443 } else {
3444 desc_buf = param_read_buf;
3445 is_kmalloc = false;
3446 }
3447
3448 /* Request for full descriptor */
3449 ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3450 desc_id, desc_index, 0,
3451 desc_buf, &buff_len);
3452
3453 if (ret) {
3454 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3455 __func__, desc_id, desc_index, param_offset, ret);
3456 goto out;
3457 }
3458
3459 /* Sanity check */
3460 if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3461 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3462 __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3463 ret = -EINVAL;
3464 goto out;
3465 }
3466
3467 /* Update descriptor length */
3468 buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3469 ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
3470
3471 if (is_kmalloc) {
3472 /* Make sure we don't copy more data than available */
3473 if (param_offset >= buff_len)
3474 ret = -EINVAL;
3475 else
3476 memcpy(param_read_buf, &desc_buf[param_offset],
3477 min_t(u32, param_size, buff_len - param_offset));
3478 }
3479 out:
3480 if (is_kmalloc)
3481 kfree(desc_buf);
3482 return ret;
3483 }
3484
3485 /**
3486 * struct uc_string_id - unicode string
3487 *
3488 * @len: size of this descriptor inclusive
3489 * @type: descriptor type
3490 * @uc: unicode string character
3491 */
3492 struct uc_string_id {
3493 u8 len;
3494 u8 type;
3495 wchar_t uc[];
3496 } __packed;
3497
3498 /* replace non-printable or non-ASCII characters with spaces */
ufshcd_remove_non_printable(u8 ch)3499 static inline char ufshcd_remove_non_printable(u8 ch)
3500 {
3501 return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3502 }
3503
3504 /**
3505 * ufshcd_read_string_desc - read string descriptor
3506 * @hba: pointer to adapter instance
3507 * @desc_index: descriptor index
3508 * @buf: pointer to buffer where descriptor would be read,
3509 * the caller should free the memory.
3510 * @ascii: if true convert from unicode to ascii characters
3511 * null terminated string.
3512 *
3513 * Return:
3514 * * string size on success.
3515 * * -ENOMEM: on allocation failure
3516 * * -EINVAL: on a wrong parameter
3517 */
ufshcd_read_string_desc(struct ufs_hba * hba,u8 desc_index,u8 ** buf,bool ascii)3518 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3519 u8 **buf, bool ascii)
3520 {
3521 struct uc_string_id *uc_str;
3522 u8 *str;
3523 int ret;
3524
3525 if (!buf)
3526 return -EINVAL;
3527
3528 uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3529 if (!uc_str)
3530 return -ENOMEM;
3531
3532 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3533 (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3534 if (ret < 0) {
3535 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3536 QUERY_REQ_RETRIES, ret);
3537 str = NULL;
3538 goto out;
3539 }
3540
3541 if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3542 dev_dbg(hba->dev, "String Desc is of zero length\n");
3543 str = NULL;
3544 ret = 0;
3545 goto out;
3546 }
3547
3548 if (ascii) {
3549 ssize_t ascii_len;
3550 int i;
3551 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3552 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3553 str = kzalloc(ascii_len, GFP_KERNEL);
3554 if (!str) {
3555 ret = -ENOMEM;
3556 goto out;
3557 }
3558
3559 /*
3560 * the descriptor contains string in UTF16 format
3561 * we need to convert to utf-8 so it can be displayed
3562 */
3563 ret = utf16s_to_utf8s(uc_str->uc,
3564 uc_str->len - QUERY_DESC_HDR_SIZE,
3565 UTF16_BIG_ENDIAN, str, ascii_len);
3566
3567 /* replace non-printable or non-ASCII characters with spaces */
3568 for (i = 0; i < ret; i++)
3569 str[i] = ufshcd_remove_non_printable(str[i]);
3570
3571 str[ret++] = '\0';
3572
3573 } else {
3574 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3575 if (!str) {
3576 ret = -ENOMEM;
3577 goto out;
3578 }
3579 ret = uc_str->len;
3580 }
3581 out:
3582 *buf = str;
3583 kfree(uc_str);
3584 return ret;
3585 }
3586
3587 /**
3588 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3589 * @hba: Pointer to adapter instance
3590 * @lun: lun id
3591 * @param_offset: offset of the parameter to read
3592 * @param_read_buf: pointer to buffer where parameter would be read
3593 * @param_size: sizeof(param_read_buf)
3594 *
3595 * Return 0 in case of success, non-zero otherwise
3596 */
ufshcd_read_unit_desc_param(struct ufs_hba * hba,int lun,enum unit_desc_param param_offset,u8 * param_read_buf,u32 param_size)3597 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3598 int lun,
3599 enum unit_desc_param param_offset,
3600 u8 *param_read_buf,
3601 u32 param_size)
3602 {
3603 /*
3604 * Unit descriptors are only available for general purpose LUs (LUN id
3605 * from 0 to 7) and RPMB Well known LU.
3606 */
3607 if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
3608 return -EOPNOTSUPP;
3609
3610 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3611 param_offset, param_read_buf, param_size);
3612 }
3613
ufshcd_get_ref_clk_gating_wait(struct ufs_hba * hba)3614 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3615 {
3616 int err = 0;
3617 u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3618
3619 if (hba->dev_info.wspecversion >= 0x300) {
3620 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3621 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3622 &gating_wait);
3623 if (err)
3624 dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3625 err, gating_wait);
3626
3627 if (gating_wait == 0) {
3628 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3629 dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3630 gating_wait);
3631 }
3632
3633 hba->dev_info.clk_gating_wait_us = gating_wait;
3634 }
3635
3636 return err;
3637 }
3638
3639 /**
3640 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3641 * @hba: per adapter instance
3642 *
3643 * 1. Allocate DMA memory for Command Descriptor array
3644 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3645 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3646 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3647 * (UTMRDL)
3648 * 4. Allocate memory for local reference block(lrb).
3649 *
3650 * Returns 0 for success, non-zero in case of failure
3651 */
ufshcd_memory_alloc(struct ufs_hba * hba)3652 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3653 {
3654 size_t utmrdl_size, utrdl_size, ucdl_size;
3655
3656 /* Allocate memory for UTP command descriptors */
3657 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3658 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3659 ucdl_size,
3660 &hba->ucdl_dma_addr,
3661 GFP_KERNEL);
3662
3663 /*
3664 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3665 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3666 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3667 * be aligned to 128 bytes as well
3668 */
3669 if (!hba->ucdl_base_addr ||
3670 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3671 dev_err(hba->dev,
3672 "Command Descriptor Memory allocation failed\n");
3673 goto out;
3674 }
3675
3676 /*
3677 * Allocate memory for UTP Transfer descriptors
3678 * UFSHCI requires 1024 byte alignment of UTRD
3679 */
3680 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3681 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3682 utrdl_size,
3683 &hba->utrdl_dma_addr,
3684 GFP_KERNEL);
3685 if (!hba->utrdl_base_addr ||
3686 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3687 dev_err(hba->dev,
3688 "Transfer Descriptor Memory allocation failed\n");
3689 goto out;
3690 }
3691
3692 /*
3693 * Allocate memory for UTP Task Management descriptors
3694 * UFSHCI requires 1024 byte alignment of UTMRD
3695 */
3696 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3697 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3698 utmrdl_size,
3699 &hba->utmrdl_dma_addr,
3700 GFP_KERNEL);
3701 if (!hba->utmrdl_base_addr ||
3702 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3703 dev_err(hba->dev,
3704 "Task Management Descriptor Memory allocation failed\n");
3705 goto out;
3706 }
3707
3708 /* Allocate memory for local reference block */
3709 hba->lrb = devm_kcalloc(hba->dev,
3710 hba->nutrs, sizeof(struct ufshcd_lrb),
3711 GFP_KERNEL);
3712 if (!hba->lrb) {
3713 dev_err(hba->dev, "LRB Memory allocation failed\n");
3714 goto out;
3715 }
3716 return 0;
3717 out:
3718 return -ENOMEM;
3719 }
3720
3721 /**
3722 * ufshcd_host_memory_configure - configure local reference block with
3723 * memory offsets
3724 * @hba: per adapter instance
3725 *
3726 * Configure Host memory space
3727 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3728 * address.
3729 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3730 * and PRDT offset.
3731 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3732 * into local reference block.
3733 */
ufshcd_host_memory_configure(struct ufs_hba * hba)3734 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3735 {
3736 struct utp_transfer_req_desc *utrdlp;
3737 dma_addr_t cmd_desc_dma_addr;
3738 dma_addr_t cmd_desc_element_addr;
3739 u16 response_offset;
3740 u16 prdt_offset;
3741 int cmd_desc_size;
3742 int i;
3743
3744 utrdlp = hba->utrdl_base_addr;
3745
3746 response_offset =
3747 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3748 prdt_offset =
3749 offsetof(struct utp_transfer_cmd_desc, prd_table);
3750
3751 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3752 cmd_desc_dma_addr = hba->ucdl_dma_addr;
3753
3754 for (i = 0; i < hba->nutrs; i++) {
3755 /* Configure UTRD with command descriptor base address */
3756 cmd_desc_element_addr =
3757 (cmd_desc_dma_addr + (cmd_desc_size * i));
3758 utrdlp[i].command_desc_base_addr_lo =
3759 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3760 utrdlp[i].command_desc_base_addr_hi =
3761 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3762
3763 /* Response upiu and prdt offset should be in double words */
3764 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3765 utrdlp[i].response_upiu_offset =
3766 cpu_to_le16(response_offset);
3767 utrdlp[i].prd_table_offset =
3768 cpu_to_le16(prdt_offset);
3769 utrdlp[i].response_upiu_length =
3770 cpu_to_le16(ALIGNED_UPIU_SIZE);
3771 } else {
3772 utrdlp[i].response_upiu_offset =
3773 cpu_to_le16(response_offset >> 2);
3774 utrdlp[i].prd_table_offset =
3775 cpu_to_le16(prdt_offset >> 2);
3776 utrdlp[i].response_upiu_length =
3777 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3778 }
3779
3780 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3781 }
3782 }
3783
3784 /**
3785 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3786 * @hba: per adapter instance
3787 *
3788 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3789 * in order to initialize the Unipro link startup procedure.
3790 * Once the Unipro links are up, the device connected to the controller
3791 * is detected.
3792 *
3793 * Returns 0 on success, non-zero value on failure
3794 */
ufshcd_dme_link_startup(struct ufs_hba * hba)3795 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3796 {
3797 struct uic_command uic_cmd = {0};
3798 int ret;
3799
3800 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3801
3802 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3803 if (ret)
3804 dev_dbg(hba->dev,
3805 "dme-link-startup: error code %d\n", ret);
3806 return ret;
3807 }
3808 /**
3809 * ufshcd_dme_reset - UIC command for DME_RESET
3810 * @hba: per adapter instance
3811 *
3812 * DME_RESET command is issued in order to reset UniPro stack.
3813 * This function now deals with cold reset.
3814 *
3815 * Returns 0 on success, non-zero value on failure
3816 */
ufshcd_dme_reset(struct ufs_hba * hba)3817 static int ufshcd_dme_reset(struct ufs_hba *hba)
3818 {
3819 struct uic_command uic_cmd = {0};
3820 int ret;
3821
3822 uic_cmd.command = UIC_CMD_DME_RESET;
3823
3824 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3825 if (ret)
3826 dev_err(hba->dev,
3827 "dme-reset: error code %d\n", ret);
3828
3829 return ret;
3830 }
3831
ufshcd_dme_configure_adapt(struct ufs_hba * hba,int agreed_gear,int adapt_val)3832 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3833 int agreed_gear,
3834 int adapt_val)
3835 {
3836 int ret;
3837
3838 if (agreed_gear < UFS_HS_G4)
3839 adapt_val = PA_NO_ADAPT;
3840
3841 ret = ufshcd_dme_set(hba,
3842 UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3843 adapt_val);
3844 return ret;
3845 }
3846 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3847
3848 /**
3849 * ufshcd_dme_enable - UIC command for DME_ENABLE
3850 * @hba: per adapter instance
3851 *
3852 * DME_ENABLE command is issued in order to enable UniPro stack.
3853 *
3854 * Returns 0 on success, non-zero value on failure
3855 */
ufshcd_dme_enable(struct ufs_hba * hba)3856 static int ufshcd_dme_enable(struct ufs_hba *hba)
3857 {
3858 struct uic_command uic_cmd = {0};
3859 int ret;
3860
3861 uic_cmd.command = UIC_CMD_DME_ENABLE;
3862
3863 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3864 if (ret)
3865 dev_err(hba->dev,
3866 "dme-enable: error code %d\n", ret);
3867
3868 return ret;
3869 }
3870
ufshcd_add_delay_before_dme_cmd(struct ufs_hba * hba)3871 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3872 {
3873 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3874 unsigned long min_sleep_time_us;
3875
3876 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3877 return;
3878
3879 /*
3880 * last_dme_cmd_tstamp will be 0 only for 1st call to
3881 * this function
3882 */
3883 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3884 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3885 } else {
3886 unsigned long delta =
3887 (unsigned long) ktime_to_us(
3888 ktime_sub(ktime_get(),
3889 hba->last_dme_cmd_tstamp));
3890
3891 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3892 min_sleep_time_us =
3893 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3894 else
3895 return; /* no more delay required */
3896 }
3897
3898 /* allow sleep for extra 50us if needed */
3899 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3900 }
3901
3902 /**
3903 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3904 * @hba: per adapter instance
3905 * @attr_sel: uic command argument1
3906 * @attr_set: attribute set type as uic command argument2
3907 * @mib_val: setting value as uic command argument3
3908 * @peer: indicate whether peer or local
3909 *
3910 * Returns 0 on success, non-zero value on failure
3911 */
ufshcd_dme_set_attr(struct ufs_hba * hba,u32 attr_sel,u8 attr_set,u32 mib_val,u8 peer)3912 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3913 u8 attr_set, u32 mib_val, u8 peer)
3914 {
3915 struct uic_command uic_cmd = {0};
3916 static const char *const action[] = {
3917 "dme-set",
3918 "dme-peer-set"
3919 };
3920 const char *set = action[!!peer];
3921 int ret;
3922 int retries = UFS_UIC_COMMAND_RETRIES;
3923
3924 uic_cmd.command = peer ?
3925 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3926 uic_cmd.argument1 = attr_sel;
3927 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3928 uic_cmd.argument3 = mib_val;
3929
3930 do {
3931 /* for peer attributes we retry upon failure */
3932 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3933 if (ret)
3934 dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3935 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3936 } while (ret && peer && --retries);
3937
3938 if (ret)
3939 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3940 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3941 UFS_UIC_COMMAND_RETRIES - retries);
3942
3943 return ret;
3944 }
3945 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3946
3947 /**
3948 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3949 * @hba: per adapter instance
3950 * @attr_sel: uic command argument1
3951 * @mib_val: the value of the attribute as returned by the UIC command
3952 * @peer: indicate whether peer or local
3953 *
3954 * Returns 0 on success, non-zero value on failure
3955 */
ufshcd_dme_get_attr(struct ufs_hba * hba,u32 attr_sel,u32 * mib_val,u8 peer)3956 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3957 u32 *mib_val, u8 peer)
3958 {
3959 struct uic_command uic_cmd = {0};
3960 static const char *const action[] = {
3961 "dme-get",
3962 "dme-peer-get"
3963 };
3964 const char *get = action[!!peer];
3965 int ret;
3966 int retries = UFS_UIC_COMMAND_RETRIES;
3967 struct ufs_pa_layer_attr orig_pwr_info;
3968 struct ufs_pa_layer_attr temp_pwr_info;
3969 bool pwr_mode_change = false;
3970
3971 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3972 orig_pwr_info = hba->pwr_info;
3973 temp_pwr_info = orig_pwr_info;
3974
3975 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3976 orig_pwr_info.pwr_rx == FAST_MODE) {
3977 temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3978 temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3979 pwr_mode_change = true;
3980 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3981 orig_pwr_info.pwr_rx == SLOW_MODE) {
3982 temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3983 temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3984 pwr_mode_change = true;
3985 }
3986 if (pwr_mode_change) {
3987 ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3988 if (ret)
3989 goto out;
3990 }
3991 }
3992
3993 uic_cmd.command = peer ?
3994 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3995 uic_cmd.argument1 = attr_sel;
3996
3997 do {
3998 /* for peer attributes we retry upon failure */
3999 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4000 if (ret)
4001 dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4002 get, UIC_GET_ATTR_ID(attr_sel), ret);
4003 } while (ret && peer && --retries);
4004
4005 if (ret)
4006 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4007 get, UIC_GET_ATTR_ID(attr_sel),
4008 UFS_UIC_COMMAND_RETRIES - retries);
4009
4010 if (mib_val && !ret)
4011 *mib_val = uic_cmd.argument3;
4012
4013 if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4014 && pwr_mode_change)
4015 ufshcd_change_power_mode(hba, &orig_pwr_info);
4016 out:
4017 return ret;
4018 }
4019 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
4020
4021 /**
4022 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4023 * state) and waits for it to take effect.
4024 *
4025 * @hba: per adapter instance
4026 * @cmd: UIC command to execute
4027 *
4028 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
4029 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
4030 * and device UniPro link and hence it's final completion would be indicated by
4031 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
4032 * addition to normal UIC command completion Status (UCCS). This function only
4033 * returns after the relevant status bits indicate the completion.
4034 *
4035 * Returns 0 on success, non-zero value on failure
4036 */
ufshcd_uic_pwr_ctrl(struct ufs_hba * hba,struct uic_command * cmd)4037 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
4038 {
4039 DECLARE_COMPLETION_ONSTACK(uic_async_done);
4040 unsigned long flags;
4041 u8 status;
4042 int ret;
4043 bool reenable_intr = false;
4044
4045 mutex_lock(&hba->uic_cmd_mutex);
4046 ufshcd_add_delay_before_dme_cmd(hba);
4047
4048 spin_lock_irqsave(hba->host->host_lock, flags);
4049 if (ufshcd_is_link_broken(hba)) {
4050 ret = -ENOLINK;
4051 goto out_unlock;
4052 }
4053 hba->uic_async_done = &uic_async_done;
4054 if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4055 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4056 /*
4057 * Make sure UIC command completion interrupt is disabled before
4058 * issuing UIC command.
4059 */
4060 wmb();
4061 reenable_intr = true;
4062 }
4063 ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4064 spin_unlock_irqrestore(hba->host->host_lock, flags);
4065 if (ret) {
4066 dev_err(hba->dev,
4067 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4068 cmd->command, cmd->argument3, ret);
4069 goto out;
4070 }
4071
4072 if (!wait_for_completion_timeout(hba->uic_async_done,
4073 msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
4074 dev_err(hba->dev,
4075 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4076 cmd->command, cmd->argument3);
4077
4078 if (!cmd->cmd_active) {
4079 dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4080 __func__);
4081 goto check_upmcrs;
4082 }
4083
4084 ret = -ETIMEDOUT;
4085 goto out;
4086 }
4087
4088 check_upmcrs:
4089 status = ufshcd_get_upmcrs(hba);
4090 if (status != PWR_LOCAL) {
4091 dev_err(hba->dev,
4092 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4093 cmd->command, status);
4094 ret = (status != PWR_OK) ? status : -1;
4095 }
4096 out:
4097 if (ret) {
4098 ufshcd_print_host_state(hba);
4099 ufshcd_print_pwr_info(hba);
4100 ufshcd_print_evt_hist(hba);
4101 }
4102
4103 spin_lock_irqsave(hba->host->host_lock, flags);
4104 hba->active_uic_cmd = NULL;
4105 hba->uic_async_done = NULL;
4106 if (reenable_intr)
4107 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4108 if (ret) {
4109 ufshcd_set_link_broken(hba);
4110 ufshcd_schedule_eh_work(hba);
4111 }
4112 out_unlock:
4113 spin_unlock_irqrestore(hba->host->host_lock, flags);
4114 mutex_unlock(&hba->uic_cmd_mutex);
4115
4116 return ret;
4117 }
4118
4119 /**
4120 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4121 * using DME_SET primitives.
4122 * @hba: per adapter instance
4123 * @mode: powr mode value
4124 *
4125 * Returns 0 on success, non-zero value on failure
4126 */
ufshcd_uic_change_pwr_mode(struct ufs_hba * hba,u8 mode)4127 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4128 {
4129 struct uic_command uic_cmd = {0};
4130 int ret;
4131
4132 if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4133 ret = ufshcd_dme_set(hba,
4134 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4135 if (ret) {
4136 dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4137 __func__, ret);
4138 goto out;
4139 }
4140 }
4141
4142 uic_cmd.command = UIC_CMD_DME_SET;
4143 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4144 uic_cmd.argument3 = mode;
4145 ufshcd_hold(hba, false);
4146 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4147 ufshcd_release(hba);
4148
4149 out:
4150 return ret;
4151 }
4152 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode);
4153
ufshcd_link_recovery(struct ufs_hba * hba)4154 int ufshcd_link_recovery(struct ufs_hba *hba)
4155 {
4156 int ret;
4157 unsigned long flags;
4158
4159 spin_lock_irqsave(hba->host->host_lock, flags);
4160 hba->ufshcd_state = UFSHCD_STATE_RESET;
4161 ufshcd_set_eh_in_progress(hba);
4162 spin_unlock_irqrestore(hba->host->host_lock, flags);
4163
4164 /* Reset the attached device */
4165 ufshcd_device_reset(hba);
4166
4167 ret = ufshcd_host_reset_and_restore(hba);
4168
4169 spin_lock_irqsave(hba->host->host_lock, flags);
4170 if (ret)
4171 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4172 ufshcd_clear_eh_in_progress(hba);
4173 spin_unlock_irqrestore(hba->host->host_lock, flags);
4174
4175 if (ret)
4176 dev_err(hba->dev, "%s: link recovery failed, err %d",
4177 __func__, ret);
4178
4179 return ret;
4180 }
4181 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4182
ufshcd_uic_hibern8_enter(struct ufs_hba * hba)4183 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4184 {
4185 int ret;
4186 struct uic_command uic_cmd = {0};
4187 ktime_t start = ktime_get();
4188
4189 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4190
4191 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
4192 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4193 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4194 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4195
4196 if (ret)
4197 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4198 __func__, ret);
4199 else
4200 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4201 POST_CHANGE);
4202
4203 return ret;
4204 }
4205 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4206
ufshcd_uic_hibern8_exit(struct ufs_hba * hba)4207 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4208 {
4209 struct uic_command uic_cmd = {0};
4210 int ret;
4211 ktime_t start = ktime_get();
4212
4213 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4214
4215 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4216 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4217 trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4218 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4219
4220 if (ret) {
4221 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4222 __func__, ret);
4223 } else {
4224 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4225 POST_CHANGE);
4226 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock();
4227 hba->ufs_stats.hibern8_exit_cnt++;
4228 }
4229
4230 return ret;
4231 }
4232 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4233
ufshcd_auto_hibern8_update(struct ufs_hba * hba,u32 ahit)4234 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4235 {
4236 unsigned long flags;
4237 bool update = false;
4238
4239 if (!ufshcd_is_auto_hibern8_supported(hba))
4240 return;
4241
4242 spin_lock_irqsave(hba->host->host_lock, flags);
4243 if (hba->ahit != ahit) {
4244 hba->ahit = ahit;
4245 update = true;
4246 }
4247 spin_unlock_irqrestore(hba->host->host_lock, flags);
4248
4249 if (update &&
4250 !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4251 ufshcd_rpm_get_sync(hba);
4252 ufshcd_hold(hba, false);
4253 ufshcd_auto_hibern8_enable(hba);
4254 ufshcd_release(hba);
4255 ufshcd_rpm_put_sync(hba);
4256 }
4257 }
4258 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4259
ufshcd_auto_hibern8_enable(struct ufs_hba * hba)4260 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
4261 {
4262 if (!ufshcd_is_auto_hibern8_supported(hba))
4263 return;
4264
4265 ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4266 }
4267
4268 /**
4269 * ufshcd_init_pwr_info - setting the POR (power on reset)
4270 * values in hba power info
4271 * @hba: per-adapter instance
4272 */
ufshcd_init_pwr_info(struct ufs_hba * hba)4273 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4274 {
4275 hba->pwr_info.gear_rx = UFS_PWM_G1;
4276 hba->pwr_info.gear_tx = UFS_PWM_G1;
4277 hba->pwr_info.lane_rx = 1;
4278 hba->pwr_info.lane_tx = 1;
4279 hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4280 hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4281 hba->pwr_info.hs_rate = 0;
4282 }
4283
4284 /**
4285 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4286 * @hba: per-adapter instance
4287 */
ufshcd_get_max_pwr_mode(struct ufs_hba * hba)4288 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4289 {
4290 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4291
4292 if (hba->max_pwr_info.is_valid)
4293 return 0;
4294
4295 if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
4296 pwr_info->pwr_tx = FASTAUTO_MODE;
4297 pwr_info->pwr_rx = FASTAUTO_MODE;
4298 } else {
4299 pwr_info->pwr_tx = FAST_MODE;
4300 pwr_info->pwr_rx = FAST_MODE;
4301 }
4302 pwr_info->hs_rate = PA_HS_MODE_B;
4303
4304 /* Get the connected lane count */
4305 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4306 &pwr_info->lane_rx);
4307 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4308 &pwr_info->lane_tx);
4309
4310 if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4311 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4312 __func__,
4313 pwr_info->lane_rx,
4314 pwr_info->lane_tx);
4315 return -EINVAL;
4316 }
4317
4318 /*
4319 * First, get the maximum gears of HS speed.
4320 * If a zero value, it means there is no HSGEAR capability.
4321 * Then, get the maximum gears of PWM speed.
4322 */
4323 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4324 if (!pwr_info->gear_rx) {
4325 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4326 &pwr_info->gear_rx);
4327 if (!pwr_info->gear_rx) {
4328 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4329 __func__, pwr_info->gear_rx);
4330 return -EINVAL;
4331 }
4332 pwr_info->pwr_rx = SLOW_MODE;
4333 }
4334
4335 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4336 &pwr_info->gear_tx);
4337 if (!pwr_info->gear_tx) {
4338 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4339 &pwr_info->gear_tx);
4340 if (!pwr_info->gear_tx) {
4341 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4342 __func__, pwr_info->gear_tx);
4343 return -EINVAL;
4344 }
4345 pwr_info->pwr_tx = SLOW_MODE;
4346 }
4347
4348 hba->max_pwr_info.is_valid = true;
4349 return 0;
4350 }
4351
ufshcd_change_power_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_mode)4352 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4353 struct ufs_pa_layer_attr *pwr_mode)
4354 {
4355 int ret;
4356
4357 /* if already configured to the requested pwr_mode */
4358 if (!hba->force_pmc &&
4359 pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4360 pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4361 pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4362 pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4363 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4364 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4365 pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4366 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4367 return 0;
4368 }
4369
4370 /*
4371 * Configure attributes for power mode change with below.
4372 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4373 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4374 * - PA_HSSERIES
4375 */
4376 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4377 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4378 pwr_mode->lane_rx);
4379 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4380 pwr_mode->pwr_rx == FAST_MODE)
4381 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
4382 else
4383 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
4384
4385 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4386 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4387 pwr_mode->lane_tx);
4388 if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4389 pwr_mode->pwr_tx == FAST_MODE)
4390 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
4391 else
4392 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
4393
4394 if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4395 pwr_mode->pwr_tx == FASTAUTO_MODE ||
4396 pwr_mode->pwr_rx == FAST_MODE ||
4397 pwr_mode->pwr_tx == FAST_MODE)
4398 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4399 pwr_mode->hs_rate);
4400
4401 if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4402 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4403 DL_FC0ProtectionTimeOutVal_Default);
4404 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4405 DL_TC0ReplayTimeOutVal_Default);
4406 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4407 DL_AFC0ReqTimeOutVal_Default);
4408 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4409 DL_FC1ProtectionTimeOutVal_Default);
4410 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4411 DL_TC1ReplayTimeOutVal_Default);
4412 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4413 DL_AFC1ReqTimeOutVal_Default);
4414
4415 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4416 DL_FC0ProtectionTimeOutVal_Default);
4417 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4418 DL_TC0ReplayTimeOutVal_Default);
4419 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4420 DL_AFC0ReqTimeOutVal_Default);
4421 }
4422
4423 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4424 | pwr_mode->pwr_tx);
4425
4426 if (ret) {
4427 dev_err(hba->dev,
4428 "%s: power mode change failed %d\n", __func__, ret);
4429 } else {
4430 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4431 pwr_mode);
4432
4433 memcpy(&hba->pwr_info, pwr_mode,
4434 sizeof(struct ufs_pa_layer_attr));
4435 }
4436
4437 return ret;
4438 }
4439
4440 /**
4441 * ufshcd_config_pwr_mode - configure a new power mode
4442 * @hba: per-adapter instance
4443 * @desired_pwr_mode: desired power configuration
4444 */
ufshcd_config_pwr_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * desired_pwr_mode)4445 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4446 struct ufs_pa_layer_attr *desired_pwr_mode)
4447 {
4448 struct ufs_pa_layer_attr final_params = { 0 };
4449 int ret;
4450
4451 ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4452 desired_pwr_mode, &final_params);
4453
4454 if (ret)
4455 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4456
4457 ret = ufshcd_change_power_mode(hba, &final_params);
4458
4459 return ret;
4460 }
4461 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4462
4463 /**
4464 * ufshcd_complete_dev_init() - checks device readiness
4465 * @hba: per-adapter instance
4466 *
4467 * Set fDeviceInit flag and poll until device toggles it.
4468 */
ufshcd_complete_dev_init(struct ufs_hba * hba)4469 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4470 {
4471 int err;
4472 bool flag_res = true;
4473 ktime_t timeout;
4474
4475 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4476 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4477 if (err) {
4478 dev_err(hba->dev,
4479 "%s setting fDeviceInit flag failed with error %d\n",
4480 __func__, err);
4481 goto out;
4482 }
4483
4484 /* Poll fDeviceInit flag to be cleared */
4485 timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4486 do {
4487 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4488 QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4489 if (!flag_res)
4490 break;
4491 usleep_range(500, 1000);
4492 } while (ktime_before(ktime_get(), timeout));
4493
4494 if (err) {
4495 dev_err(hba->dev,
4496 "%s reading fDeviceInit flag failed with error %d\n",
4497 __func__, err);
4498 } else if (flag_res) {
4499 dev_err(hba->dev,
4500 "%s fDeviceInit was not cleared by the device\n",
4501 __func__);
4502 err = -EBUSY;
4503 }
4504 out:
4505 return err;
4506 }
4507
4508 /**
4509 * ufshcd_make_hba_operational - Make UFS controller operational
4510 * @hba: per adapter instance
4511 *
4512 * To bring UFS host controller to operational state,
4513 * 1. Enable required interrupts
4514 * 2. Configure interrupt aggregation
4515 * 3. Program UTRL and UTMRL base address
4516 * 4. Configure run-stop-registers
4517 *
4518 * Returns 0 on success, non-zero value on failure
4519 */
ufshcd_make_hba_operational(struct ufs_hba * hba)4520 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4521 {
4522 int err = 0;
4523 u32 reg;
4524
4525 /* Enable required interrupts */
4526 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4527
4528 /* Configure interrupt aggregation */
4529 if (ufshcd_is_intr_aggr_allowed(hba))
4530 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4531 else
4532 ufshcd_disable_intr_aggr(hba);
4533
4534 /* Configure UTRL and UTMRL base address registers */
4535 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4536 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4537 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4538 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4539 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4540 REG_UTP_TASK_REQ_LIST_BASE_L);
4541 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4542 REG_UTP_TASK_REQ_LIST_BASE_H);
4543
4544 /*
4545 * Make sure base address and interrupt setup are updated before
4546 * enabling the run/stop registers below.
4547 */
4548 wmb();
4549
4550 /*
4551 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4552 */
4553 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4554 if (!(ufshcd_get_lists_status(reg))) {
4555 ufshcd_enable_run_stop_reg(hba);
4556 } else {
4557 dev_err(hba->dev,
4558 "Host controller not ready to process requests");
4559 err = -EIO;
4560 }
4561
4562 return err;
4563 }
4564 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4565
4566 /**
4567 * ufshcd_hba_stop - Send controller to reset state
4568 * @hba: per adapter instance
4569 */
ufshcd_hba_stop(struct ufs_hba * hba)4570 void ufshcd_hba_stop(struct ufs_hba *hba)
4571 {
4572 unsigned long flags;
4573 int err;
4574
4575 /*
4576 * Obtain the host lock to prevent that the controller is disabled
4577 * while the UFS interrupt handler is active on another CPU.
4578 */
4579 spin_lock_irqsave(hba->host->host_lock, flags);
4580 ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
4581 spin_unlock_irqrestore(hba->host->host_lock, flags);
4582
4583 err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4584 CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4585 10, 1);
4586 if (err)
4587 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4588 }
4589 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4590
4591 /**
4592 * ufshcd_hba_execute_hce - initialize the controller
4593 * @hba: per adapter instance
4594 *
4595 * The controller resets itself and controller firmware initialization
4596 * sequence kicks off. When controller is ready it will set
4597 * the Host Controller Enable bit to 1.
4598 *
4599 * Returns 0 on success, non-zero value on failure
4600 */
ufshcd_hba_execute_hce(struct ufs_hba * hba)4601 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4602 {
4603 int retry_outer = 3;
4604 int retry_inner;
4605
4606 start:
4607 if (ufshcd_is_hba_active(hba))
4608 /* change controller state to "reset state" */
4609 ufshcd_hba_stop(hba);
4610
4611 /* UniPro link is disabled at this point */
4612 ufshcd_set_link_off(hba);
4613
4614 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4615
4616 /* start controller initialization sequence */
4617 ufshcd_hba_start(hba);
4618
4619 /*
4620 * To initialize a UFS host controller HCE bit must be set to 1.
4621 * During initialization the HCE bit value changes from 1->0->1.
4622 * When the host controller completes initialization sequence
4623 * it sets the value of HCE bit to 1. The same HCE bit is read back
4624 * to check if the controller has completed initialization sequence.
4625 * So without this delay the value HCE = 1, set in the previous
4626 * instruction might be read back.
4627 * This delay can be changed based on the controller.
4628 */
4629 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4630
4631 /* wait for the host controller to complete initialization */
4632 retry_inner = 50;
4633 while (!ufshcd_is_hba_active(hba)) {
4634 if (retry_inner) {
4635 retry_inner--;
4636 } else {
4637 dev_err(hba->dev,
4638 "Controller enable failed\n");
4639 if (retry_outer) {
4640 retry_outer--;
4641 goto start;
4642 }
4643 return -EIO;
4644 }
4645 usleep_range(1000, 1100);
4646 }
4647
4648 /* enable UIC related interrupts */
4649 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4650
4651 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4652
4653 return 0;
4654 }
4655
ufshcd_hba_enable(struct ufs_hba * hba)4656 int ufshcd_hba_enable(struct ufs_hba *hba)
4657 {
4658 int ret;
4659
4660 if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4661 ufshcd_set_link_off(hba);
4662 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4663
4664 /* enable UIC related interrupts */
4665 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4666 ret = ufshcd_dme_reset(hba);
4667 if (!ret) {
4668 ret = ufshcd_dme_enable(hba);
4669 if (!ret)
4670 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4671 if (ret)
4672 dev_err(hba->dev,
4673 "Host controller enable failed with non-hce\n");
4674 }
4675 } else {
4676 ret = ufshcd_hba_execute_hce(hba);
4677 }
4678
4679 return ret;
4680 }
4681 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4682
ufshcd_disable_tx_lcc(struct ufs_hba * hba,bool peer)4683 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4684 {
4685 int tx_lanes = 0, i, err = 0;
4686
4687 if (!peer)
4688 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4689 &tx_lanes);
4690 else
4691 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4692 &tx_lanes);
4693 for (i = 0; i < tx_lanes; i++) {
4694 if (!peer)
4695 err = ufshcd_dme_set(hba,
4696 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4697 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4698 0);
4699 else
4700 err = ufshcd_dme_peer_set(hba,
4701 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4702 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4703 0);
4704 if (err) {
4705 dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4706 __func__, peer, i, err);
4707 break;
4708 }
4709 }
4710
4711 return err;
4712 }
4713
ufshcd_disable_device_tx_lcc(struct ufs_hba * hba)4714 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4715 {
4716 return ufshcd_disable_tx_lcc(hba, true);
4717 }
4718
ufshcd_update_evt_hist(struct ufs_hba * hba,u32 id,u32 val)4719 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4720 {
4721 struct ufs_event_hist *e;
4722
4723 if (id >= UFS_EVT_CNT)
4724 return;
4725
4726 e = &hba->ufs_stats.event[id];
4727 e->val[e->pos] = val;
4728 e->tstamp[e->pos] = local_clock();
4729 e->cnt += 1;
4730 e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4731
4732 ufshcd_vops_event_notify(hba, id, &val);
4733 }
4734 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4735
4736 /**
4737 * ufshcd_link_startup - Initialize unipro link startup
4738 * @hba: per adapter instance
4739 *
4740 * Returns 0 for success, non-zero in case of failure
4741 */
ufshcd_link_startup(struct ufs_hba * hba)4742 static int ufshcd_link_startup(struct ufs_hba *hba)
4743 {
4744 int ret;
4745 int retries = DME_LINKSTARTUP_RETRIES;
4746 bool link_startup_again = false;
4747
4748 /*
4749 * If UFS device isn't active then we will have to issue link startup
4750 * 2 times to make sure the device state move to active.
4751 */
4752 if (!ufshcd_is_ufs_dev_active(hba))
4753 link_startup_again = true;
4754
4755 link_startup:
4756 do {
4757 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4758
4759 ret = ufshcd_dme_link_startup(hba);
4760
4761 /* check if device is detected by inter-connect layer */
4762 if (!ret && !ufshcd_is_device_present(hba)) {
4763 ufshcd_update_evt_hist(hba,
4764 UFS_EVT_LINK_STARTUP_FAIL,
4765 0);
4766 dev_err(hba->dev, "%s: Device not present\n", __func__);
4767 ret = -ENXIO;
4768 goto out;
4769 }
4770
4771 /*
4772 * DME link lost indication is only received when link is up,
4773 * but we can't be sure if the link is up until link startup
4774 * succeeds. So reset the local Uni-Pro and try again.
4775 */
4776 if (ret && retries && ufshcd_hba_enable(hba)) {
4777 ufshcd_update_evt_hist(hba,
4778 UFS_EVT_LINK_STARTUP_FAIL,
4779 (u32)ret);
4780 goto out;
4781 }
4782 } while (ret && retries--);
4783
4784 if (ret) {
4785 /* failed to get the link up... retire */
4786 ufshcd_update_evt_hist(hba,
4787 UFS_EVT_LINK_STARTUP_FAIL,
4788 (u32)ret);
4789 goto out;
4790 }
4791
4792 if (link_startup_again) {
4793 link_startup_again = false;
4794 retries = DME_LINKSTARTUP_RETRIES;
4795 goto link_startup;
4796 }
4797
4798 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4799 ufshcd_init_pwr_info(hba);
4800 ufshcd_print_pwr_info(hba);
4801
4802 if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4803 ret = ufshcd_disable_device_tx_lcc(hba);
4804 if (ret)
4805 goto out;
4806 }
4807
4808 /* Include any host controller configuration via UIC commands */
4809 ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4810 if (ret)
4811 goto out;
4812
4813 /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4814 ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
4815 ret = ufshcd_make_hba_operational(hba);
4816 out:
4817 if (ret) {
4818 dev_err(hba->dev, "link startup failed %d\n", ret);
4819 ufshcd_print_host_state(hba);
4820 ufshcd_print_pwr_info(hba);
4821 ufshcd_print_evt_hist(hba);
4822 }
4823 return ret;
4824 }
4825
4826 /**
4827 * ufshcd_verify_dev_init() - Verify device initialization
4828 * @hba: per-adapter instance
4829 *
4830 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4831 * device Transport Protocol (UTP) layer is ready after a reset.
4832 * If the UTP layer at the device side is not initialized, it may
4833 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4834 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4835 */
ufshcd_verify_dev_init(struct ufs_hba * hba)4836 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4837 {
4838 int err = 0;
4839 int retries;
4840
4841 ufshcd_hold(hba, false);
4842 mutex_lock(&hba->dev_cmd.lock);
4843 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4844 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4845 hba->nop_out_timeout);
4846
4847 if (!err || err == -ETIMEDOUT)
4848 break;
4849
4850 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4851 }
4852 mutex_unlock(&hba->dev_cmd.lock);
4853 ufshcd_release(hba);
4854
4855 if (err)
4856 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4857 return err;
4858 }
4859
4860 /**
4861 * ufshcd_set_queue_depth - set lun queue depth
4862 * @sdev: pointer to SCSI device
4863 *
4864 * Read bLUQueueDepth value and activate scsi tagged command
4865 * queueing. For WLUN, queue depth is set to 1. For best-effort
4866 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4867 * value that host can queue.
4868 */
ufshcd_set_queue_depth(struct scsi_device * sdev)4869 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4870 {
4871 int ret = 0;
4872 u8 lun_qdepth;
4873 struct ufs_hba *hba;
4874
4875 hba = shost_priv(sdev->host);
4876
4877 lun_qdepth = hba->nutrs;
4878 ret = ufshcd_read_unit_desc_param(hba,
4879 ufshcd_scsi_to_upiu_lun(sdev->lun),
4880 UNIT_DESC_PARAM_LU_Q_DEPTH,
4881 &lun_qdepth,
4882 sizeof(lun_qdepth));
4883
4884 /* Some WLUN doesn't support unit descriptor */
4885 if (ret == -EOPNOTSUPP)
4886 lun_qdepth = 1;
4887 else if (!lun_qdepth)
4888 /* eventually, we can figure out the real queue depth */
4889 lun_qdepth = hba->nutrs;
4890 else
4891 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4892
4893 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4894 __func__, lun_qdepth);
4895 scsi_change_queue_depth(sdev, lun_qdepth);
4896 }
4897
4898 /*
4899 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4900 * @hba: per-adapter instance
4901 * @lun: UFS device lun id
4902 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4903 *
4904 * Returns 0 in case of success and b_lu_write_protect status would be returned
4905 * @b_lu_write_protect parameter.
4906 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4907 * Returns -EINVAL in case of invalid parameters passed to this function.
4908 */
ufshcd_get_lu_wp(struct ufs_hba * hba,u8 lun,u8 * b_lu_write_protect)4909 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4910 u8 lun,
4911 u8 *b_lu_write_protect)
4912 {
4913 int ret;
4914
4915 if (!b_lu_write_protect)
4916 ret = -EINVAL;
4917 /*
4918 * According to UFS device spec, RPMB LU can't be write
4919 * protected so skip reading bLUWriteProtect parameter for
4920 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4921 */
4922 else if (lun >= hba->dev_info.max_lu_supported)
4923 ret = -ENOTSUPP;
4924 else
4925 ret = ufshcd_read_unit_desc_param(hba,
4926 lun,
4927 UNIT_DESC_PARAM_LU_WR_PROTECT,
4928 b_lu_write_protect,
4929 sizeof(*b_lu_write_protect));
4930 return ret;
4931 }
4932
4933 /**
4934 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4935 * status
4936 * @hba: per-adapter instance
4937 * @sdev: pointer to SCSI device
4938 *
4939 */
ufshcd_get_lu_power_on_wp_status(struct ufs_hba * hba,const struct scsi_device * sdev)4940 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4941 const struct scsi_device *sdev)
4942 {
4943 if (hba->dev_info.f_power_on_wp_en &&
4944 !hba->dev_info.is_lu_power_on_wp) {
4945 u8 b_lu_write_protect;
4946
4947 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4948 &b_lu_write_protect) &&
4949 (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4950 hba->dev_info.is_lu_power_on_wp = true;
4951 }
4952 }
4953
4954 /**
4955 * ufshcd_setup_links - associate link b/w device wlun and other luns
4956 * @sdev: pointer to SCSI device
4957 * @hba: pointer to ufs hba
4958 */
ufshcd_setup_links(struct ufs_hba * hba,struct scsi_device * sdev)4959 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
4960 {
4961 struct device_link *link;
4962
4963 /*
4964 * Device wlun is the supplier & rest of the luns are consumers.
4965 * This ensures that device wlun suspends after all other luns.
4966 */
4967 if (hba->ufs_device_wlun) {
4968 link = device_link_add(&sdev->sdev_gendev,
4969 &hba->ufs_device_wlun->sdev_gendev,
4970 DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
4971 if (!link) {
4972 dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
4973 dev_name(&hba->ufs_device_wlun->sdev_gendev));
4974 return;
4975 }
4976 hba->luns_avail--;
4977 /* Ignore REPORT_LUN wlun probing */
4978 if (hba->luns_avail == 1) {
4979 ufshcd_rpm_put(hba);
4980 return;
4981 }
4982 } else {
4983 /*
4984 * Device wlun is probed. The assumption is that WLUNs are
4985 * scanned before other LUNs.
4986 */
4987 hba->luns_avail--;
4988 }
4989 }
4990
4991 /**
4992 * ufshcd_slave_alloc - handle initial SCSI device configurations
4993 * @sdev: pointer to SCSI device
4994 *
4995 * Returns success
4996 */
ufshcd_slave_alloc(struct scsi_device * sdev)4997 static int ufshcd_slave_alloc(struct scsi_device *sdev)
4998 {
4999 struct ufs_hba *hba;
5000
5001 hba = shost_priv(sdev->host);
5002
5003 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
5004 sdev->use_10_for_ms = 1;
5005
5006 /* DBD field should be set to 1 in mode sense(10) */
5007 sdev->set_dbd_for_ms = 1;
5008
5009 /* allow SCSI layer to restart the device in case of errors */
5010 sdev->allow_restart = 1;
5011
5012 /* REPORT SUPPORTED OPERATION CODES is not supported */
5013 sdev->no_report_opcodes = 1;
5014
5015 /* WRITE_SAME command is not supported */
5016 sdev->no_write_same = 1;
5017
5018 ufshcd_set_queue_depth(sdev);
5019
5020 ufshcd_get_lu_power_on_wp_status(hba, sdev);
5021
5022 ufshcd_setup_links(hba, sdev);
5023
5024 return 0;
5025 }
5026
5027 /**
5028 * ufshcd_change_queue_depth - change queue depth
5029 * @sdev: pointer to SCSI device
5030 * @depth: required depth to set
5031 *
5032 * Change queue depth and make sure the max. limits are not crossed.
5033 */
ufshcd_change_queue_depth(struct scsi_device * sdev,int depth)5034 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
5035 {
5036 return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5037 }
5038
ufshcd_hpb_destroy(struct ufs_hba * hba,struct scsi_device * sdev)5039 static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev)
5040 {
5041 /* skip well-known LU */
5042 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
5043 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
5044 return;
5045
5046 ufshpb_destroy_lu(hba, sdev);
5047 }
5048
ufshcd_hpb_configure(struct ufs_hba * hba,struct scsi_device * sdev)5049 static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev)
5050 {
5051 /* skip well-known LU */
5052 if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) ||
5053 !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba))
5054 return;
5055
5056 ufshpb_init_hpb_lu(hba, sdev);
5057 }
5058
5059 /**
5060 * ufshcd_slave_configure - adjust SCSI device configurations
5061 * @sdev: pointer to SCSI device
5062 */
ufshcd_slave_configure(struct scsi_device * sdev)5063 static int ufshcd_slave_configure(struct scsi_device *sdev)
5064 {
5065 struct ufs_hba *hba = shost_priv(sdev->host);
5066 struct request_queue *q = sdev->request_queue;
5067
5068 ufshcd_hpb_configure(hba, sdev);
5069
5070 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
5071 if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
5072 blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
5073 /*
5074 * Block runtime-pm until all consumers are added.
5075 * Refer ufshcd_setup_links().
5076 */
5077 if (is_device_wlun(sdev))
5078 pm_runtime_get_noresume(&sdev->sdev_gendev);
5079 else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5080 sdev->rpm_autosuspend = 1;
5081 /*
5082 * Do not print messages during runtime PM to avoid never-ending cycles
5083 * of messages written back to storage by user space causing runtime
5084 * resume, causing more messages and so on.
5085 */
5086 sdev->silence_suspend = 1;
5087
5088 ufshcd_crypto_register(hba, q);
5089
5090 return 0;
5091 }
5092
5093 /**
5094 * ufshcd_slave_destroy - remove SCSI device configurations
5095 * @sdev: pointer to SCSI device
5096 */
ufshcd_slave_destroy(struct scsi_device * sdev)5097 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5098 {
5099 struct ufs_hba *hba;
5100 unsigned long flags;
5101
5102 hba = shost_priv(sdev->host);
5103
5104 ufshcd_hpb_destroy(hba, sdev);
5105
5106 /* Drop the reference as it won't be needed anymore */
5107 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5108 spin_lock_irqsave(hba->host->host_lock, flags);
5109 hba->ufs_device_wlun = NULL;
5110 spin_unlock_irqrestore(hba->host->host_lock, flags);
5111 } else if (hba->ufs_device_wlun) {
5112 struct device *supplier = NULL;
5113
5114 /* Ensure UFS Device WLUN exists and does not disappear */
5115 spin_lock_irqsave(hba->host->host_lock, flags);
5116 if (hba->ufs_device_wlun) {
5117 supplier = &hba->ufs_device_wlun->sdev_gendev;
5118 get_device(supplier);
5119 }
5120 spin_unlock_irqrestore(hba->host->host_lock, flags);
5121
5122 if (supplier) {
5123 /*
5124 * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5125 * device will not have been registered but can still
5126 * have a device link holding a reference to the device.
5127 */
5128 device_link_remove(&sdev->sdev_gendev, supplier);
5129 put_device(supplier);
5130 }
5131 }
5132 }
5133
5134 /**
5135 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5136 * @lrbp: pointer to local reference block of completed command
5137 * @scsi_status: SCSI command status
5138 *
5139 * Returns value base on SCSI command status
5140 */
5141 static inline int
ufshcd_scsi_cmd_status(struct ufshcd_lrb * lrbp,int scsi_status)5142 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5143 {
5144 int result = 0;
5145
5146 switch (scsi_status) {
5147 case SAM_STAT_CHECK_CONDITION:
5148 ufshcd_copy_sense_data(lrbp);
5149 fallthrough;
5150 case SAM_STAT_GOOD:
5151 result |= DID_OK << 16 | scsi_status;
5152 break;
5153 case SAM_STAT_TASK_SET_FULL:
5154 case SAM_STAT_BUSY:
5155 case SAM_STAT_TASK_ABORTED:
5156 ufshcd_copy_sense_data(lrbp);
5157 result |= scsi_status;
5158 break;
5159 default:
5160 result |= DID_ERROR << 16;
5161 break;
5162 } /* end of switch */
5163
5164 return result;
5165 }
5166
5167 /**
5168 * ufshcd_transfer_rsp_status - Get overall status of the response
5169 * @hba: per adapter instance
5170 * @lrbp: pointer to local reference block of completed command
5171 *
5172 * Returns result of the command to notify SCSI midlayer
5173 */
5174 static inline int
ufshcd_transfer_rsp_status(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)5175 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
5176 {
5177 int result = 0;
5178 int scsi_status;
5179 enum utp_ocs ocs;
5180
5181 /* overall command status of utrd */
5182 ocs = ufshcd_get_tr_ocs(lrbp);
5183
5184 if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5185 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
5186 MASK_RSP_UPIU_RESULT)
5187 ocs = OCS_SUCCESS;
5188 }
5189
5190 switch (ocs) {
5191 case OCS_SUCCESS:
5192 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
5193 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5194 switch (result) {
5195 case UPIU_TRANSACTION_RESPONSE:
5196 /*
5197 * get the response UPIU result to extract
5198 * the SCSI command status
5199 */
5200 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
5201
5202 /*
5203 * get the result based on SCSI status response
5204 * to notify the SCSI midlayer of the command status
5205 */
5206 scsi_status = result & MASK_SCSI_STATUS;
5207 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5208
5209 /*
5210 * Currently we are only supporting BKOPs exception
5211 * events hence we can ignore BKOPs exception event
5212 * during power management callbacks. BKOPs exception
5213 * event is not expected to be raised in runtime suspend
5214 * callback as it allows the urgent bkops.
5215 * During system suspend, we are anyway forcefully
5216 * disabling the bkops and if urgent bkops is needed
5217 * it will be enabled on system resume. Long term
5218 * solution could be to abort the system suspend if
5219 * UFS device needs urgent BKOPs.
5220 */
5221 if (!hba->pm_op_in_progress &&
5222 !ufshcd_eh_in_progress(hba) &&
5223 ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5224 /* Flushed in suspend */
5225 schedule_work(&hba->eeh_work);
5226
5227 if (scsi_status == SAM_STAT_GOOD)
5228 ufshpb_rsp_upiu(hba, lrbp);
5229 break;
5230 case UPIU_TRANSACTION_REJECT_UPIU:
5231 /* TODO: handle Reject UPIU Response */
5232 result = DID_ERROR << 16;
5233 dev_err(hba->dev,
5234 "Reject UPIU not fully implemented\n");
5235 break;
5236 default:
5237 dev_err(hba->dev,
5238 "Unexpected request response code = %x\n",
5239 result);
5240 result = DID_ERROR << 16;
5241 break;
5242 }
5243 break;
5244 case OCS_ABORTED:
5245 result |= DID_ABORT << 16;
5246 break;
5247 case OCS_INVALID_COMMAND_STATUS:
5248 result |= DID_REQUEUE << 16;
5249 break;
5250 case OCS_INVALID_CMD_TABLE_ATTR:
5251 case OCS_INVALID_PRDT_ATTR:
5252 case OCS_MISMATCH_DATA_BUF_SIZE:
5253 case OCS_MISMATCH_RESP_UPIU_SIZE:
5254 case OCS_PEER_COMM_FAILURE:
5255 case OCS_FATAL_ERROR:
5256 case OCS_DEVICE_FATAL_ERROR:
5257 case OCS_INVALID_CRYPTO_CONFIG:
5258 case OCS_GENERAL_CRYPTO_ERROR:
5259 default:
5260 result |= DID_ERROR << 16;
5261 dev_err(hba->dev,
5262 "OCS error from controller = %x for tag %d\n",
5263 ocs, lrbp->task_tag);
5264 ufshcd_print_evt_hist(hba);
5265 ufshcd_print_host_state(hba);
5266 break;
5267 } /* end of switch */
5268
5269 if ((host_byte(result) != DID_OK) &&
5270 (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5271 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
5272 return result;
5273 }
5274
ufshcd_is_auto_hibern8_error(struct ufs_hba * hba,u32 intr_mask)5275 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5276 u32 intr_mask)
5277 {
5278 if (!ufshcd_is_auto_hibern8_supported(hba) ||
5279 !ufshcd_is_auto_hibern8_enabled(hba))
5280 return false;
5281
5282 if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5283 return false;
5284
5285 if (hba->active_uic_cmd &&
5286 (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5287 hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5288 return false;
5289
5290 return true;
5291 }
5292
5293 /**
5294 * ufshcd_uic_cmd_compl - handle completion of uic command
5295 * @hba: per adapter instance
5296 * @intr_status: interrupt status generated by the controller
5297 *
5298 * Returns
5299 * IRQ_HANDLED - If interrupt is valid
5300 * IRQ_NONE - If invalid interrupt
5301 */
ufshcd_uic_cmd_compl(struct ufs_hba * hba,u32 intr_status)5302 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5303 {
5304 irqreturn_t retval = IRQ_NONE;
5305
5306 spin_lock(hba->host->host_lock);
5307 if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5308 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5309
5310 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5311 hba->active_uic_cmd->argument2 |=
5312 ufshcd_get_uic_cmd_result(hba);
5313 hba->active_uic_cmd->argument3 =
5314 ufshcd_get_dme_attr_val(hba);
5315 if (!hba->uic_async_done)
5316 hba->active_uic_cmd->cmd_active = 0;
5317 complete(&hba->active_uic_cmd->done);
5318 retval = IRQ_HANDLED;
5319 }
5320
5321 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5322 hba->active_uic_cmd->cmd_active = 0;
5323 complete(hba->uic_async_done);
5324 retval = IRQ_HANDLED;
5325 }
5326
5327 if (retval == IRQ_HANDLED)
5328 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5329 UFS_CMD_COMP);
5330 spin_unlock(hba->host->host_lock);
5331 return retval;
5332 }
5333
5334 /* Release the resources allocated for processing a SCSI command. */
ufshcd_release_scsi_cmd(struct ufs_hba * hba,struct ufshcd_lrb * lrbp)5335 static void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5336 struct ufshcd_lrb *lrbp)
5337 {
5338 struct scsi_cmnd *cmd = lrbp->cmd;
5339
5340 scsi_dma_unmap(cmd);
5341 lrbp->cmd = NULL; /* Mark the command as completed. */
5342 ufshcd_release(hba);
5343 ufshcd_clk_scaling_update_busy(hba);
5344 }
5345
5346 /**
5347 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5348 * @hba: per adapter instance
5349 * @completed_reqs: bitmask that indicates which requests to complete
5350 */
__ufshcd_transfer_req_compl(struct ufs_hba * hba,unsigned long completed_reqs)5351 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5352 unsigned long completed_reqs)
5353 {
5354 struct ufshcd_lrb *lrbp;
5355 struct scsi_cmnd *cmd;
5356 int index;
5357
5358 for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5359 lrbp = &hba->lrb[index];
5360 lrbp->compl_time_stamp = ktime_get();
5361 lrbp->compl_time_stamp_local_clock = local_clock();
5362 cmd = lrbp->cmd;
5363 if (cmd) {
5364 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5365 ufshcd_update_monitor(hba, lrbp);
5366 ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
5367 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp);
5368 ufshcd_release_scsi_cmd(hba, lrbp);
5369 /* Do not touch lrbp after scsi done */
5370 scsi_done(cmd);
5371 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5372 lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5373 if (hba->dev_cmd.complete) {
5374 ufshcd_add_command_trace(hba, index,
5375 UFS_DEV_COMP);
5376 complete(hba->dev_cmd.complete);
5377 ufshcd_clk_scaling_update_busy(hba);
5378 }
5379 }
5380 }
5381 }
5382
5383 /* Any value that is not an existing queue number is fine for this constant. */
5384 enum {
5385 UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1
5386 };
5387
ufshcd_clear_polled(struct ufs_hba * hba,unsigned long * completed_reqs)5388 static void ufshcd_clear_polled(struct ufs_hba *hba,
5389 unsigned long *completed_reqs)
5390 {
5391 int tag;
5392
5393 for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5394 struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5395
5396 if (!cmd)
5397 continue;
5398 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED)
5399 __clear_bit(tag, completed_reqs);
5400 }
5401 }
5402
5403 /*
5404 * Returns > 0 if one or more commands have been completed or 0 if no
5405 * requests have been completed.
5406 */
ufshcd_poll(struct Scsi_Host * shost,unsigned int queue_num)5407 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5408 {
5409 struct ufs_hba *hba = shost_priv(shost);
5410 unsigned long completed_reqs, flags;
5411 u32 tr_doorbell;
5412
5413 spin_lock_irqsave(&hba->outstanding_lock, flags);
5414 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5415 completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5416 WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5417 "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5418 hba->outstanding_reqs);
5419 if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) {
5420 /* Do not complete polled requests from interrupt context. */
5421 ufshcd_clear_polled(hba, &completed_reqs);
5422 }
5423 hba->outstanding_reqs &= ~completed_reqs;
5424 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5425
5426 if (completed_reqs)
5427 __ufshcd_transfer_req_compl(hba, completed_reqs);
5428
5429 return completed_reqs != 0;
5430 }
5431
5432 /**
5433 * ufshcd_transfer_req_compl - handle SCSI and query command completion
5434 * @hba: per adapter instance
5435 *
5436 * Returns
5437 * IRQ_HANDLED - If interrupt is valid
5438 * IRQ_NONE - If invalid interrupt
5439 */
ufshcd_transfer_req_compl(struct ufs_hba * hba)5440 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5441 {
5442 /* Resetting interrupt aggregation counters first and reading the
5443 * DOOR_BELL afterward allows us to handle all the completed requests.
5444 * In order to prevent other interrupts starvation the DB is read once
5445 * after reset. The down side of this solution is the possibility of
5446 * false interrupt if device completes another request after resetting
5447 * aggregation and before reading the DB.
5448 */
5449 if (ufshcd_is_intr_aggr_allowed(hba) &&
5450 !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5451 ufshcd_reset_intr_aggr(hba);
5452
5453 if (ufs_fail_completion())
5454 return IRQ_HANDLED;
5455
5456 /*
5457 * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5458 * do not want polling to trigger spurious interrupt complaints.
5459 */
5460 ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
5461
5462 return IRQ_HANDLED;
5463 }
5464
__ufshcd_write_ee_control(struct ufs_hba * hba,u32 ee_ctrl_mask)5465 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5466 {
5467 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5468 QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5469 &ee_ctrl_mask);
5470 }
5471
ufshcd_write_ee_control(struct ufs_hba * hba)5472 int ufshcd_write_ee_control(struct ufs_hba *hba)
5473 {
5474 int err;
5475
5476 mutex_lock(&hba->ee_ctrl_mutex);
5477 err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5478 mutex_unlock(&hba->ee_ctrl_mutex);
5479 if (err)
5480 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5481 __func__, err);
5482 return err;
5483 }
5484
ufshcd_update_ee_control(struct ufs_hba * hba,u16 * mask,const u16 * other_mask,u16 set,u16 clr)5485 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask,
5486 const u16 *other_mask, u16 set, u16 clr)
5487 {
5488 u16 new_mask, ee_ctrl_mask;
5489 int err = 0;
5490
5491 mutex_lock(&hba->ee_ctrl_mutex);
5492 new_mask = (*mask & ~clr) | set;
5493 ee_ctrl_mask = new_mask | *other_mask;
5494 if (ee_ctrl_mask != hba->ee_ctrl_mask)
5495 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5496 /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5497 if (!err) {
5498 hba->ee_ctrl_mask = ee_ctrl_mask;
5499 *mask = new_mask;
5500 }
5501 mutex_unlock(&hba->ee_ctrl_mutex);
5502 return err;
5503 }
5504
5505 /**
5506 * ufshcd_disable_ee - disable exception event
5507 * @hba: per-adapter instance
5508 * @mask: exception event to disable
5509 *
5510 * Disables exception event in the device so that the EVENT_ALERT
5511 * bit is not set.
5512 *
5513 * Returns zero on success, non-zero error value on failure.
5514 */
ufshcd_disable_ee(struct ufs_hba * hba,u16 mask)5515 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5516 {
5517 return ufshcd_update_ee_drv_mask(hba, 0, mask);
5518 }
5519
5520 /**
5521 * ufshcd_enable_ee - enable exception event
5522 * @hba: per-adapter instance
5523 * @mask: exception event to enable
5524 *
5525 * Enable corresponding exception event in the device to allow
5526 * device to alert host in critical scenarios.
5527 *
5528 * Returns zero on success, non-zero error value on failure.
5529 */
ufshcd_enable_ee(struct ufs_hba * hba,u16 mask)5530 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5531 {
5532 return ufshcd_update_ee_drv_mask(hba, mask, 0);
5533 }
5534
5535 /**
5536 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5537 * @hba: per-adapter instance
5538 *
5539 * Allow device to manage background operations on its own. Enabling
5540 * this might lead to inconsistent latencies during normal data transfers
5541 * as the device is allowed to manage its own way of handling background
5542 * operations.
5543 *
5544 * Returns zero on success, non-zero on failure.
5545 */
ufshcd_enable_auto_bkops(struct ufs_hba * hba)5546 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5547 {
5548 int err = 0;
5549
5550 if (hba->auto_bkops_enabled)
5551 goto out;
5552
5553 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5554 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5555 if (err) {
5556 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5557 __func__, err);
5558 goto out;
5559 }
5560
5561 hba->auto_bkops_enabled = true;
5562 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5563
5564 /* No need of URGENT_BKOPS exception from the device */
5565 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5566 if (err)
5567 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5568 __func__, err);
5569 out:
5570 return err;
5571 }
5572
5573 /**
5574 * ufshcd_disable_auto_bkops - block device in doing background operations
5575 * @hba: per-adapter instance
5576 *
5577 * Disabling background operations improves command response latency but
5578 * has drawback of device moving into critical state where the device is
5579 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5580 * host is idle so that BKOPS are managed effectively without any negative
5581 * impacts.
5582 *
5583 * Returns zero on success, non-zero on failure.
5584 */
ufshcd_disable_auto_bkops(struct ufs_hba * hba)5585 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5586 {
5587 int err = 0;
5588
5589 if (!hba->auto_bkops_enabled)
5590 goto out;
5591
5592 /*
5593 * If host assisted BKOPs is to be enabled, make sure
5594 * urgent bkops exception is allowed.
5595 */
5596 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5597 if (err) {
5598 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5599 __func__, err);
5600 goto out;
5601 }
5602
5603 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5604 QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5605 if (err) {
5606 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5607 __func__, err);
5608 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5609 goto out;
5610 }
5611
5612 hba->auto_bkops_enabled = false;
5613 trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5614 hba->is_urgent_bkops_lvl_checked = false;
5615 out:
5616 return err;
5617 }
5618
5619 /**
5620 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5621 * @hba: per adapter instance
5622 *
5623 * After a device reset the device may toggle the BKOPS_EN flag
5624 * to default value. The s/w tracking variables should be updated
5625 * as well. This function would change the auto-bkops state based on
5626 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5627 */
ufshcd_force_reset_auto_bkops(struct ufs_hba * hba)5628 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5629 {
5630 if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5631 hba->auto_bkops_enabled = false;
5632 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5633 ufshcd_enable_auto_bkops(hba);
5634 } else {
5635 hba->auto_bkops_enabled = true;
5636 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5637 ufshcd_disable_auto_bkops(hba);
5638 }
5639 hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5640 hba->is_urgent_bkops_lvl_checked = false;
5641 }
5642
ufshcd_get_bkops_status(struct ufs_hba * hba,u32 * status)5643 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5644 {
5645 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5646 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5647 }
5648
5649 /**
5650 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5651 * @hba: per-adapter instance
5652 * @status: bkops_status value
5653 *
5654 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5655 * flag in the device to permit background operations if the device
5656 * bkops_status is greater than or equal to "status" argument passed to
5657 * this function, disable otherwise.
5658 *
5659 * Returns 0 for success, non-zero in case of failure.
5660 *
5661 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5662 * to know whether auto bkops is enabled or disabled after this function
5663 * returns control to it.
5664 */
ufshcd_bkops_ctrl(struct ufs_hba * hba,enum bkops_status status)5665 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5666 enum bkops_status status)
5667 {
5668 int err;
5669 u32 curr_status = 0;
5670
5671 err = ufshcd_get_bkops_status(hba, &curr_status);
5672 if (err) {
5673 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5674 __func__, err);
5675 goto out;
5676 } else if (curr_status > BKOPS_STATUS_MAX) {
5677 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5678 __func__, curr_status);
5679 err = -EINVAL;
5680 goto out;
5681 }
5682
5683 if (curr_status >= status)
5684 err = ufshcd_enable_auto_bkops(hba);
5685 else
5686 err = ufshcd_disable_auto_bkops(hba);
5687 out:
5688 return err;
5689 }
5690
5691 /**
5692 * ufshcd_urgent_bkops - handle urgent bkops exception event
5693 * @hba: per-adapter instance
5694 *
5695 * Enable fBackgroundOpsEn flag in the device to permit background
5696 * operations.
5697 *
5698 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5699 * and negative error value for any other failure.
5700 */
ufshcd_urgent_bkops(struct ufs_hba * hba)5701 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5702 {
5703 return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5704 }
5705
ufshcd_get_ee_status(struct ufs_hba * hba,u32 * status)5706 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5707 {
5708 return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5709 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5710 }
5711
ufshcd_bkops_exception_event_handler(struct ufs_hba * hba)5712 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5713 {
5714 int err;
5715 u32 curr_status = 0;
5716
5717 if (hba->is_urgent_bkops_lvl_checked)
5718 goto enable_auto_bkops;
5719
5720 err = ufshcd_get_bkops_status(hba, &curr_status);
5721 if (err) {
5722 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5723 __func__, err);
5724 goto out;
5725 }
5726
5727 /*
5728 * We are seeing that some devices are raising the urgent bkops
5729 * exception events even when BKOPS status doesn't indicate performace
5730 * impacted or critical. Handle these device by determining their urgent
5731 * bkops status at runtime.
5732 */
5733 if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5734 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5735 __func__, curr_status);
5736 /* update the current status as the urgent bkops level */
5737 hba->urgent_bkops_lvl = curr_status;
5738 hba->is_urgent_bkops_lvl_checked = true;
5739 }
5740
5741 enable_auto_bkops:
5742 err = ufshcd_enable_auto_bkops(hba);
5743 out:
5744 if (err < 0)
5745 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5746 __func__, err);
5747 }
5748
ufshcd_temp_exception_event_handler(struct ufs_hba * hba,u16 status)5749 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5750 {
5751 u32 value;
5752
5753 if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5754 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
5755 return;
5756
5757 dev_info(hba->dev, "exception Tcase %d\n", value - 80);
5758
5759 ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
5760
5761 /*
5762 * A placeholder for the platform vendors to add whatever additional
5763 * steps required
5764 */
5765 }
5766
__ufshcd_wb_toggle(struct ufs_hba * hba,bool set,enum flag_idn idn)5767 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
5768 {
5769 u8 index;
5770 enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5771 UPIU_QUERY_OPCODE_CLEAR_FLAG;
5772
5773 index = ufshcd_wb_get_query_index(hba);
5774 return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
5775 }
5776
ufshcd_wb_toggle(struct ufs_hba * hba,bool enable)5777 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
5778 {
5779 int ret;
5780
5781 if (!ufshcd_is_wb_allowed(hba) ||
5782 hba->dev_info.wb_enabled == enable)
5783 return 0;
5784
5785 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
5786 if (ret) {
5787 dev_err(hba->dev, "%s: Write Booster %s failed %d\n",
5788 __func__, enable ? "enabling" : "disabling", ret);
5789 return ret;
5790 }
5791
5792 hba->dev_info.wb_enabled = enable;
5793 dev_dbg(hba->dev, "%s: Write Booster %s\n",
5794 __func__, enable ? "enabled" : "disabled");
5795
5796 return ret;
5797 }
5798
ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba * hba,bool enable)5799 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
5800 bool enable)
5801 {
5802 int ret;
5803
5804 ret = __ufshcd_wb_toggle(hba, enable,
5805 QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
5806 if (ret) {
5807 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n",
5808 __func__, enable ? "enabling" : "disabling", ret);
5809 return;
5810 }
5811 dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n",
5812 __func__, enable ? "enabled" : "disabled");
5813 }
5814
ufshcd_wb_toggle_buf_flush(struct ufs_hba * hba,bool enable)5815 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable)
5816 {
5817 int ret;
5818
5819 if (!ufshcd_is_wb_allowed(hba) ||
5820 hba->dev_info.wb_buf_flush_enabled == enable)
5821 return 0;
5822
5823 ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
5824 if (ret) {
5825 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n",
5826 __func__, enable ? "enabling" : "disabling", ret);
5827 return ret;
5828 }
5829
5830 hba->dev_info.wb_buf_flush_enabled = enable;
5831 dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n",
5832 __func__, enable ? "enabled" : "disabled");
5833
5834 return ret;
5835 }
5836
ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba * hba,u32 avail_buf)5837 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5838 u32 avail_buf)
5839 {
5840 u32 cur_buf;
5841 int ret;
5842 u8 index;
5843
5844 index = ufshcd_wb_get_query_index(hba);
5845 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5846 QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
5847 index, 0, &cur_buf);
5848 if (ret) {
5849 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n",
5850 __func__, ret);
5851 return false;
5852 }
5853
5854 if (!cur_buf) {
5855 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5856 cur_buf);
5857 return false;
5858 }
5859 /* Let it continue to flush when available buffer exceeds threshold */
5860 return avail_buf < hba->vps->wb_flush_threshold;
5861 }
5862
ufshcd_wb_force_disable(struct ufs_hba * hba)5863 static void ufshcd_wb_force_disable(struct ufs_hba *hba)
5864 {
5865 if (ufshcd_is_wb_buf_flush_allowed(hba))
5866 ufshcd_wb_toggle_buf_flush(hba, false);
5867
5868 ufshcd_wb_toggle_buf_flush_during_h8(hba, false);
5869 ufshcd_wb_toggle(hba, false);
5870 hba->caps &= ~UFSHCD_CAP_WB_EN;
5871
5872 dev_info(hba->dev, "%s: WB force disabled\n", __func__);
5873 }
5874
ufshcd_is_wb_buf_lifetime_available(struct ufs_hba * hba)5875 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
5876 {
5877 u32 lifetime;
5878 int ret;
5879 u8 index;
5880
5881 index = ufshcd_wb_get_query_index(hba);
5882 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5883 QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
5884 index, 0, &lifetime);
5885 if (ret) {
5886 dev_err(hba->dev,
5887 "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
5888 __func__, ret);
5889 return false;
5890 }
5891
5892 if (lifetime == UFS_WB_EXCEED_LIFETIME) {
5893 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
5894 __func__, lifetime);
5895 return false;
5896 }
5897
5898 dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
5899 __func__, lifetime);
5900
5901 return true;
5902 }
5903
ufshcd_wb_need_flush(struct ufs_hba * hba)5904 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
5905 {
5906 int ret;
5907 u32 avail_buf;
5908 u8 index;
5909
5910 if (!ufshcd_is_wb_allowed(hba))
5911 return false;
5912
5913 if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
5914 ufshcd_wb_force_disable(hba);
5915 return false;
5916 }
5917
5918 /*
5919 * The ufs device needs the vcc to be ON to flush.
5920 * With user-space reduction enabled, it's enough to enable flush
5921 * by checking only the available buffer. The threshold
5922 * defined here is > 90% full.
5923 * With user-space preserved enabled, the current-buffer
5924 * should be checked too because the wb buffer size can reduce
5925 * when disk tends to be full. This info is provided by current
5926 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5927 * keeping vcc on when current buffer is empty.
5928 */
5929 index = ufshcd_wb_get_query_index(hba);
5930 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5931 QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
5932 index, 0, &avail_buf);
5933 if (ret) {
5934 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n",
5935 __func__, ret);
5936 return false;
5937 }
5938
5939 if (!hba->dev_info.b_presrv_uspc_en)
5940 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
5941
5942 return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5943 }
5944
ufshcd_rpm_dev_flush_recheck_work(struct work_struct * work)5945 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5946 {
5947 struct ufs_hba *hba = container_of(to_delayed_work(work),
5948 struct ufs_hba,
5949 rpm_dev_flush_recheck_work);
5950 /*
5951 * To prevent unnecessary VCC power drain after device finishes
5952 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5953 * after a certain delay to recheck the threshold by next runtime
5954 * suspend.
5955 */
5956 ufshcd_rpm_get_sync(hba);
5957 ufshcd_rpm_put_sync(hba);
5958 }
5959
5960 /**
5961 * ufshcd_exception_event_handler - handle exceptions raised by device
5962 * @work: pointer to work data
5963 *
5964 * Read bExceptionEventStatus attribute from the device and handle the
5965 * exception event accordingly.
5966 */
ufshcd_exception_event_handler(struct work_struct * work)5967 static void ufshcd_exception_event_handler(struct work_struct *work)
5968 {
5969 struct ufs_hba *hba;
5970 int err;
5971 u32 status = 0;
5972 hba = container_of(work, struct ufs_hba, eeh_work);
5973
5974 ufshcd_scsi_block_requests(hba);
5975 err = ufshcd_get_ee_status(hba, &status);
5976 if (err) {
5977 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5978 __func__, err);
5979 goto out;
5980 }
5981
5982 trace_ufshcd_exception_event(dev_name(hba->dev), status);
5983
5984 if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
5985 ufshcd_bkops_exception_event_handler(hba);
5986
5987 if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
5988 ufshcd_temp_exception_event_handler(hba, status);
5989
5990 ufs_debugfs_exception_event(hba, status);
5991 out:
5992 ufshcd_scsi_unblock_requests(hba);
5993 }
5994
5995 /* Complete requests that have door-bell cleared */
ufshcd_complete_requests(struct ufs_hba * hba)5996 static void ufshcd_complete_requests(struct ufs_hba *hba)
5997 {
5998 ufshcd_transfer_req_compl(hba);
5999 ufshcd_tmc_handler(hba);
6000 }
6001
6002 /**
6003 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
6004 * to recover from the DL NAC errors or not.
6005 * @hba: per-adapter instance
6006 *
6007 * Returns true if error handling is required, false otherwise
6008 */
ufshcd_quirk_dl_nac_errors(struct ufs_hba * hba)6009 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
6010 {
6011 unsigned long flags;
6012 bool err_handling = true;
6013
6014 spin_lock_irqsave(hba->host->host_lock, flags);
6015 /*
6016 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
6017 * device fatal error and/or DL NAC & REPLAY timeout errors.
6018 */
6019 if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
6020 goto out;
6021
6022 if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
6023 ((hba->saved_err & UIC_ERROR) &&
6024 (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
6025 goto out;
6026
6027 if ((hba->saved_err & UIC_ERROR) &&
6028 (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
6029 int err;
6030 /*
6031 * wait for 50ms to see if we can get any other errors or not.
6032 */
6033 spin_unlock_irqrestore(hba->host->host_lock, flags);
6034 msleep(50);
6035 spin_lock_irqsave(hba->host->host_lock, flags);
6036
6037 /*
6038 * now check if we have got any other severe errors other than
6039 * DL NAC error?
6040 */
6041 if ((hba->saved_err & INT_FATAL_ERRORS) ||
6042 ((hba->saved_err & UIC_ERROR) &&
6043 (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6044 goto out;
6045
6046 /*
6047 * As DL NAC is the only error received so far, send out NOP
6048 * command to confirm if link is still active or not.
6049 * - If we don't get any response then do error recovery.
6050 * - If we get response then clear the DL NAC error bit.
6051 */
6052
6053 spin_unlock_irqrestore(hba->host->host_lock, flags);
6054 err = ufshcd_verify_dev_init(hba);
6055 spin_lock_irqsave(hba->host->host_lock, flags);
6056
6057 if (err)
6058 goto out;
6059
6060 /* Link seems to be alive hence ignore the DL NAC errors */
6061 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6062 hba->saved_err &= ~UIC_ERROR;
6063 /* clear NAC error */
6064 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6065 if (!hba->saved_uic_err)
6066 err_handling = false;
6067 }
6068 out:
6069 spin_unlock_irqrestore(hba->host->host_lock, flags);
6070 return err_handling;
6071 }
6072
6073 /* host lock must be held before calling this func */
ufshcd_is_saved_err_fatal(struct ufs_hba * hba)6074 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6075 {
6076 return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6077 (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6078 }
6079
ufshcd_schedule_eh_work(struct ufs_hba * hba)6080 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
6081 {
6082 lockdep_assert_held(hba->host->host_lock);
6083
6084 /* handle fatal errors only when link is not in error state */
6085 if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6086 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6087 ufshcd_is_saved_err_fatal(hba))
6088 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6089 else
6090 hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6091 queue_work(hba->eh_wq, &hba->eh_work);
6092 }
6093 }
6094
ufshcd_force_error_recovery(struct ufs_hba * hba)6095 static void ufshcd_force_error_recovery(struct ufs_hba *hba)
6096 {
6097 spin_lock_irq(hba->host->host_lock);
6098 hba->force_reset = true;
6099 ufshcd_schedule_eh_work(hba);
6100 spin_unlock_irq(hba->host->host_lock);
6101 }
6102
ufshcd_clk_scaling_allow(struct ufs_hba * hba,bool allow)6103 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6104 {
6105 mutex_lock(&hba->wb_mutex);
6106 down_write(&hba->clk_scaling_lock);
6107 hba->clk_scaling.is_allowed = allow;
6108 up_write(&hba->clk_scaling_lock);
6109 mutex_unlock(&hba->wb_mutex);
6110 }
6111
ufshcd_clk_scaling_suspend(struct ufs_hba * hba,bool suspend)6112 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6113 {
6114 if (suspend) {
6115 if (hba->clk_scaling.is_enabled)
6116 ufshcd_suspend_clkscaling(hba);
6117 ufshcd_clk_scaling_allow(hba, false);
6118 } else {
6119 ufshcd_clk_scaling_allow(hba, true);
6120 if (hba->clk_scaling.is_enabled)
6121 ufshcd_resume_clkscaling(hba);
6122 }
6123 }
6124
ufshcd_err_handling_prepare(struct ufs_hba * hba)6125 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6126 {
6127 ufshcd_rpm_get_sync(hba);
6128 if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6129 hba->is_sys_suspended) {
6130 enum ufs_pm_op pm_op;
6131
6132 /*
6133 * Don't assume anything of resume, if
6134 * resume fails, irq and clocks can be OFF, and powers
6135 * can be OFF or in LPM.
6136 */
6137 ufshcd_setup_hba_vreg(hba, true);
6138 ufshcd_enable_irq(hba);
6139 ufshcd_setup_vreg(hba, true);
6140 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6141 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6142 ufshcd_hold(hba, false);
6143 if (!ufshcd_is_clkgating_allowed(hba))
6144 ufshcd_setup_clocks(hba, true);
6145 ufshcd_release(hba);
6146 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6147 ufshcd_vops_resume(hba, pm_op);
6148 } else {
6149 ufshcd_hold(hba, false);
6150 if (ufshcd_is_clkscaling_supported(hba) &&
6151 hba->clk_scaling.is_enabled)
6152 ufshcd_suspend_clkscaling(hba);
6153 ufshcd_clk_scaling_allow(hba, false);
6154 }
6155 ufshcd_scsi_block_requests(hba);
6156 /* Drain ufshcd_queuecommand() */
6157 synchronize_rcu();
6158 cancel_work_sync(&hba->eeh_work);
6159 }
6160
ufshcd_err_handling_unprepare(struct ufs_hba * hba)6161 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6162 {
6163 ufshcd_scsi_unblock_requests(hba);
6164 ufshcd_release(hba);
6165 if (ufshcd_is_clkscaling_supported(hba))
6166 ufshcd_clk_scaling_suspend(hba, false);
6167 ufshcd_rpm_put(hba);
6168 }
6169
ufshcd_err_handling_should_stop(struct ufs_hba * hba)6170 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6171 {
6172 return (!hba->is_powered || hba->shutting_down ||
6173 !hba->ufs_device_wlun ||
6174 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6175 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6176 ufshcd_is_link_broken(hba))));
6177 }
6178
6179 #ifdef CONFIG_PM
ufshcd_recover_pm_error(struct ufs_hba * hba)6180 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6181 {
6182 struct Scsi_Host *shost = hba->host;
6183 struct scsi_device *sdev;
6184 struct request_queue *q;
6185 int ret;
6186
6187 hba->is_sys_suspended = false;
6188 /*
6189 * Set RPM status of wlun device to RPM_ACTIVE,
6190 * this also clears its runtime error.
6191 */
6192 ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6193
6194 /* hba device might have a runtime error otherwise */
6195 if (ret)
6196 ret = pm_runtime_set_active(hba->dev);
6197 /*
6198 * If wlun device had runtime error, we also need to resume those
6199 * consumer scsi devices in case any of them has failed to be
6200 * resumed due to supplier runtime resume failure. This is to unblock
6201 * blk_queue_enter in case there are bios waiting inside it.
6202 */
6203 if (!ret) {
6204 shost_for_each_device(sdev, shost) {
6205 q = sdev->request_queue;
6206 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6207 q->rpm_status == RPM_SUSPENDING))
6208 pm_request_resume(q->dev);
6209 }
6210 }
6211 }
6212 #else
ufshcd_recover_pm_error(struct ufs_hba * hba)6213 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6214 {
6215 }
6216 #endif
6217
ufshcd_is_pwr_mode_restore_needed(struct ufs_hba * hba)6218 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6219 {
6220 struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6221 u32 mode;
6222
6223 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6224
6225 if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6226 return true;
6227
6228 if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6229 return true;
6230
6231 return false;
6232 }
6233
6234 /**
6235 * ufshcd_err_handler - handle UFS errors that require s/w attention
6236 * @work: pointer to work structure
6237 */
ufshcd_err_handler(struct work_struct * work)6238 static void ufshcd_err_handler(struct work_struct *work)
6239 {
6240 int retries = MAX_ERR_HANDLER_RETRIES;
6241 struct ufs_hba *hba;
6242 unsigned long flags;
6243 bool needs_restore;
6244 bool needs_reset;
6245 bool err_xfer;
6246 bool err_tm;
6247 int pmc_err;
6248 int tag;
6249
6250 hba = container_of(work, struct ufs_hba, eh_work);
6251
6252 dev_info(hba->dev,
6253 "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6254 __func__, ufshcd_state_name[hba->ufshcd_state],
6255 hba->is_powered, hba->shutting_down, hba->saved_err,
6256 hba->saved_uic_err, hba->force_reset,
6257 ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6258
6259 down(&hba->host_sem);
6260 spin_lock_irqsave(hba->host->host_lock, flags);
6261 if (ufshcd_err_handling_should_stop(hba)) {
6262 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6263 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6264 spin_unlock_irqrestore(hba->host->host_lock, flags);
6265 up(&hba->host_sem);
6266 return;
6267 }
6268 ufshcd_set_eh_in_progress(hba);
6269 spin_unlock_irqrestore(hba->host->host_lock, flags);
6270 ufshcd_err_handling_prepare(hba);
6271 /* Complete requests that have door-bell cleared by h/w */
6272 ufshcd_complete_requests(hba);
6273 spin_lock_irqsave(hba->host->host_lock, flags);
6274 again:
6275 needs_restore = false;
6276 needs_reset = false;
6277 err_xfer = false;
6278 err_tm = false;
6279
6280 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6281 hba->ufshcd_state = UFSHCD_STATE_RESET;
6282 /*
6283 * A full reset and restore might have happened after preparation
6284 * is finished, double check whether we should stop.
6285 */
6286 if (ufshcd_err_handling_should_stop(hba))
6287 goto skip_err_handling;
6288
6289 if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6290 bool ret;
6291
6292 spin_unlock_irqrestore(hba->host->host_lock, flags);
6293 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6294 ret = ufshcd_quirk_dl_nac_errors(hba);
6295 spin_lock_irqsave(hba->host->host_lock, flags);
6296 if (!ret && ufshcd_err_handling_should_stop(hba))
6297 goto skip_err_handling;
6298 }
6299
6300 if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6301 (hba->saved_uic_err &&
6302 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6303 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6304
6305 spin_unlock_irqrestore(hba->host->host_lock, flags);
6306 ufshcd_print_host_state(hba);
6307 ufshcd_print_pwr_info(hba);
6308 ufshcd_print_evt_hist(hba);
6309 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6310 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
6311 spin_lock_irqsave(hba->host->host_lock, flags);
6312 }
6313
6314 /*
6315 * if host reset is required then skip clearing the pending
6316 * transfers forcefully because they will get cleared during
6317 * host reset and restore
6318 */
6319 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6320 ufshcd_is_saved_err_fatal(hba) ||
6321 ((hba->saved_err & UIC_ERROR) &&
6322 (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6323 UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6324 needs_reset = true;
6325 goto do_reset;
6326 }
6327
6328 /*
6329 * If LINERESET was caught, UFS might have been put to PWM mode,
6330 * check if power mode restore is needed.
6331 */
6332 if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6333 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6334 if (!hba->saved_uic_err)
6335 hba->saved_err &= ~UIC_ERROR;
6336 spin_unlock_irqrestore(hba->host->host_lock, flags);
6337 if (ufshcd_is_pwr_mode_restore_needed(hba))
6338 needs_restore = true;
6339 spin_lock_irqsave(hba->host->host_lock, flags);
6340 if (!hba->saved_err && !needs_restore)
6341 goto skip_err_handling;
6342 }
6343
6344 hba->silence_err_logs = true;
6345 /* release lock as clear command might sleep */
6346 spin_unlock_irqrestore(hba->host->host_lock, flags);
6347 /* Clear pending transfer requests */
6348 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
6349 if (ufshcd_try_to_abort_task(hba, tag)) {
6350 err_xfer = true;
6351 goto lock_skip_pending_xfer_clear;
6352 }
6353 dev_err(hba->dev, "Aborted tag %d / CDB %#02x\n", tag,
6354 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1);
6355 }
6356
6357 /* Clear pending task management requests */
6358 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6359 if (ufshcd_clear_tm_cmd(hba, tag)) {
6360 err_tm = true;
6361 goto lock_skip_pending_xfer_clear;
6362 }
6363 }
6364
6365 lock_skip_pending_xfer_clear:
6366 /* Complete the requests that are cleared by s/w */
6367 ufshcd_complete_requests(hba);
6368
6369 spin_lock_irqsave(hba->host->host_lock, flags);
6370 hba->silence_err_logs = false;
6371 if (err_xfer || err_tm) {
6372 needs_reset = true;
6373 goto do_reset;
6374 }
6375
6376 /*
6377 * After all reqs and tasks are cleared from doorbell,
6378 * now it is safe to retore power mode.
6379 */
6380 if (needs_restore) {
6381 spin_unlock_irqrestore(hba->host->host_lock, flags);
6382 /*
6383 * Hold the scaling lock just in case dev cmds
6384 * are sent via bsg and/or sysfs.
6385 */
6386 down_write(&hba->clk_scaling_lock);
6387 hba->force_pmc = true;
6388 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6389 if (pmc_err) {
6390 needs_reset = true;
6391 dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6392 __func__, pmc_err);
6393 }
6394 hba->force_pmc = false;
6395 ufshcd_print_pwr_info(hba);
6396 up_write(&hba->clk_scaling_lock);
6397 spin_lock_irqsave(hba->host->host_lock, flags);
6398 }
6399
6400 do_reset:
6401 /* Fatal errors need reset */
6402 if (needs_reset) {
6403 int err;
6404
6405 hba->force_reset = false;
6406 spin_unlock_irqrestore(hba->host->host_lock, flags);
6407 err = ufshcd_reset_and_restore(hba);
6408 if (err)
6409 dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6410 __func__, err);
6411 else
6412 ufshcd_recover_pm_error(hba);
6413 spin_lock_irqsave(hba->host->host_lock, flags);
6414 }
6415
6416 skip_err_handling:
6417 if (!needs_reset) {
6418 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6419 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6420 if (hba->saved_err || hba->saved_uic_err)
6421 dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6422 __func__, hba->saved_err, hba->saved_uic_err);
6423 }
6424 /* Exit in an operational state or dead */
6425 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6426 hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6427 if (--retries)
6428 goto again;
6429 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6430 }
6431 ufshcd_clear_eh_in_progress(hba);
6432 spin_unlock_irqrestore(hba->host->host_lock, flags);
6433 ufshcd_err_handling_unprepare(hba);
6434 up(&hba->host_sem);
6435
6436 dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6437 ufshcd_state_name[hba->ufshcd_state]);
6438 }
6439
6440 /**
6441 * ufshcd_update_uic_error - check and set fatal UIC error flags.
6442 * @hba: per-adapter instance
6443 *
6444 * Returns
6445 * IRQ_HANDLED - If interrupt is valid
6446 * IRQ_NONE - If invalid interrupt
6447 */
ufshcd_update_uic_error(struct ufs_hba * hba)6448 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6449 {
6450 u32 reg;
6451 irqreturn_t retval = IRQ_NONE;
6452
6453 /* PHY layer error */
6454 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6455 if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6456 (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6457 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6458 /*
6459 * To know whether this error is fatal or not, DB timeout
6460 * must be checked but this error is handled separately.
6461 */
6462 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6463 dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6464 __func__);
6465
6466 /* Got a LINERESET indication. */
6467 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6468 struct uic_command *cmd = NULL;
6469
6470 hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6471 if (hba->uic_async_done && hba->active_uic_cmd)
6472 cmd = hba->active_uic_cmd;
6473 /*
6474 * Ignore the LINERESET during power mode change
6475 * operation via DME_SET command.
6476 */
6477 if (cmd && (cmd->command == UIC_CMD_DME_SET))
6478 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6479 }
6480 retval |= IRQ_HANDLED;
6481 }
6482
6483 /* PA_INIT_ERROR is fatal and needs UIC reset */
6484 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6485 if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6486 (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6487 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6488
6489 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6490 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6491 else if (hba->dev_quirks &
6492 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6493 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6494 hba->uic_error |=
6495 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6496 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6497 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6498 }
6499 retval |= IRQ_HANDLED;
6500 }
6501
6502 /* UIC NL/TL/DME errors needs software retry */
6503 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6504 if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6505 (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6506 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6507 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6508 retval |= IRQ_HANDLED;
6509 }
6510
6511 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6512 if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6513 (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6514 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6515 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6516 retval |= IRQ_HANDLED;
6517 }
6518
6519 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6520 if ((reg & UIC_DME_ERROR) &&
6521 (reg & UIC_DME_ERROR_CODE_MASK)) {
6522 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6523 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6524 retval |= IRQ_HANDLED;
6525 }
6526
6527 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6528 __func__, hba->uic_error);
6529 return retval;
6530 }
6531
6532 /**
6533 * ufshcd_check_errors - Check for errors that need s/w attention
6534 * @hba: per-adapter instance
6535 * @intr_status: interrupt status generated by the controller
6536 *
6537 * Returns
6538 * IRQ_HANDLED - If interrupt is valid
6539 * IRQ_NONE - If invalid interrupt
6540 */
ufshcd_check_errors(struct ufs_hba * hba,u32 intr_status)6541 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6542 {
6543 bool queue_eh_work = false;
6544 irqreturn_t retval = IRQ_NONE;
6545
6546 spin_lock(hba->host->host_lock);
6547 hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6548
6549 if (hba->errors & INT_FATAL_ERRORS) {
6550 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6551 hba->errors);
6552 queue_eh_work = true;
6553 }
6554
6555 if (hba->errors & UIC_ERROR) {
6556 hba->uic_error = 0;
6557 retval = ufshcd_update_uic_error(hba);
6558 if (hba->uic_error)
6559 queue_eh_work = true;
6560 }
6561
6562 if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6563 dev_err(hba->dev,
6564 "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6565 __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6566 "Enter" : "Exit",
6567 hba->errors, ufshcd_get_upmcrs(hba));
6568 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6569 hba->errors);
6570 ufshcd_set_link_broken(hba);
6571 queue_eh_work = true;
6572 }
6573
6574 if (queue_eh_work) {
6575 /*
6576 * update the transfer error masks to sticky bits, let's do this
6577 * irrespective of current ufshcd_state.
6578 */
6579 hba->saved_err |= hba->errors;
6580 hba->saved_uic_err |= hba->uic_error;
6581
6582 /* dump controller state before resetting */
6583 if ((hba->saved_err &
6584 (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6585 (hba->saved_uic_err &&
6586 (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6587 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6588 __func__, hba->saved_err,
6589 hba->saved_uic_err);
6590 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6591 "host_regs: ");
6592 ufshcd_print_pwr_info(hba);
6593 }
6594 ufshcd_schedule_eh_work(hba);
6595 retval |= IRQ_HANDLED;
6596 }
6597 /*
6598 * if (!queue_eh_work) -
6599 * Other errors are either non-fatal where host recovers
6600 * itself without s/w intervention or errors that will be
6601 * handled by the SCSI core layer.
6602 */
6603 hba->errors = 0;
6604 hba->uic_error = 0;
6605 spin_unlock(hba->host->host_lock);
6606 return retval;
6607 }
6608
6609 /**
6610 * ufshcd_tmc_handler - handle task management function completion
6611 * @hba: per adapter instance
6612 *
6613 * Returns
6614 * IRQ_HANDLED - If interrupt is valid
6615 * IRQ_NONE - If invalid interrupt
6616 */
ufshcd_tmc_handler(struct ufs_hba * hba)6617 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6618 {
6619 unsigned long flags, pending, issued;
6620 irqreturn_t ret = IRQ_NONE;
6621 int tag;
6622
6623 spin_lock_irqsave(hba->host->host_lock, flags);
6624 pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6625 issued = hba->outstanding_tasks & ~pending;
6626 for_each_set_bit(tag, &issued, hba->nutmrs) {
6627 struct request *req = hba->tmf_rqs[tag];
6628 struct completion *c = req->end_io_data;
6629
6630 complete(c);
6631 ret = IRQ_HANDLED;
6632 }
6633 spin_unlock_irqrestore(hba->host->host_lock, flags);
6634
6635 return ret;
6636 }
6637
6638 /**
6639 * ufshcd_sl_intr - Interrupt service routine
6640 * @hba: per adapter instance
6641 * @intr_status: contains interrupts generated by the controller
6642 *
6643 * Returns
6644 * IRQ_HANDLED - If interrupt is valid
6645 * IRQ_NONE - If invalid interrupt
6646 */
ufshcd_sl_intr(struct ufs_hba * hba,u32 intr_status)6647 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6648 {
6649 irqreturn_t retval = IRQ_NONE;
6650
6651 if (intr_status & UFSHCD_UIC_MASK)
6652 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6653
6654 if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6655 retval |= ufshcd_check_errors(hba, intr_status);
6656
6657 if (intr_status & UTP_TASK_REQ_COMPL)
6658 retval |= ufshcd_tmc_handler(hba);
6659
6660 if (intr_status & UTP_TRANSFER_REQ_COMPL)
6661 retval |= ufshcd_transfer_req_compl(hba);
6662
6663 return retval;
6664 }
6665
6666 /**
6667 * ufshcd_intr - Main interrupt service routine
6668 * @irq: irq number
6669 * @__hba: pointer to adapter instance
6670 *
6671 * Returns
6672 * IRQ_HANDLED - If interrupt is valid
6673 * IRQ_NONE - If invalid interrupt
6674 */
ufshcd_intr(int irq,void * __hba)6675 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6676 {
6677 u32 intr_status, enabled_intr_status = 0;
6678 irqreturn_t retval = IRQ_NONE;
6679 struct ufs_hba *hba = __hba;
6680 int retries = hba->nutrs;
6681
6682 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6683 hba->ufs_stats.last_intr_status = intr_status;
6684 hba->ufs_stats.last_intr_ts = local_clock();
6685
6686 /*
6687 * There could be max of hba->nutrs reqs in flight and in worst case
6688 * if the reqs get finished 1 by 1 after the interrupt status is
6689 * read, make sure we handle them by checking the interrupt status
6690 * again in a loop until we process all of the reqs before returning.
6691 */
6692 while (intr_status && retries--) {
6693 enabled_intr_status =
6694 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6695 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6696 if (enabled_intr_status)
6697 retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6698
6699 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6700 }
6701
6702 if (enabled_intr_status && retval == IRQ_NONE &&
6703 (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6704 hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6705 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6706 __func__,
6707 intr_status,
6708 hba->ufs_stats.last_intr_status,
6709 enabled_intr_status);
6710 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6711 }
6712
6713 return retval;
6714 }
6715
ufshcd_clear_tm_cmd(struct ufs_hba * hba,int tag)6716 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6717 {
6718 int err = 0;
6719 u32 mask = 1 << tag;
6720 unsigned long flags;
6721
6722 if (!test_bit(tag, &hba->outstanding_tasks))
6723 goto out;
6724
6725 spin_lock_irqsave(hba->host->host_lock, flags);
6726 ufshcd_utmrl_clear(hba, tag);
6727 spin_unlock_irqrestore(hba->host->host_lock, flags);
6728
6729 /* poll for max. 1 sec to clear door bell register by h/w */
6730 err = ufshcd_wait_for_register(hba,
6731 REG_UTP_TASK_REQ_DOOR_BELL,
6732 mask, 0, 1000, 1000);
6733
6734 dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
6735 tag, err ? "succeeded" : "failed");
6736
6737 out:
6738 return err;
6739 }
6740
__ufshcd_issue_tm_cmd(struct ufs_hba * hba,struct utp_task_req_desc * treq,u8 tm_function)6741 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6742 struct utp_task_req_desc *treq, u8 tm_function)
6743 {
6744 struct request_queue *q = hba->tmf_queue;
6745 struct Scsi_Host *host = hba->host;
6746 DECLARE_COMPLETION_ONSTACK(wait);
6747 struct request *req;
6748 unsigned long flags;
6749 int task_tag, err;
6750
6751 /*
6752 * blk_mq_alloc_request() is used here only to get a free tag.
6753 */
6754 req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
6755 if (IS_ERR(req))
6756 return PTR_ERR(req);
6757
6758 req->end_io_data = &wait;
6759 ufshcd_hold(hba, false);
6760
6761 spin_lock_irqsave(host->host_lock, flags);
6762
6763 task_tag = req->tag;
6764 WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n",
6765 task_tag);
6766 hba->tmf_rqs[req->tag] = req;
6767 treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag);
6768
6769 memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
6770 ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
6771
6772 /* send command to the controller */
6773 __set_bit(task_tag, &hba->outstanding_tasks);
6774
6775 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
6776 /* Make sure that doorbell is committed immediately */
6777 wmb();
6778
6779 spin_unlock_irqrestore(host->host_lock, flags);
6780
6781 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
6782
6783 /* wait until the task management command is completed */
6784 err = wait_for_completion_io_timeout(&wait,
6785 msecs_to_jiffies(TM_CMD_TIMEOUT));
6786 if (!err) {
6787 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
6788 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6789 __func__, tm_function);
6790 if (ufshcd_clear_tm_cmd(hba, task_tag))
6791 dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
6792 __func__, task_tag);
6793 err = -ETIMEDOUT;
6794 } else {
6795 err = 0;
6796 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
6797
6798 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
6799 }
6800
6801 spin_lock_irqsave(hba->host->host_lock, flags);
6802 hba->tmf_rqs[req->tag] = NULL;
6803 __clear_bit(task_tag, &hba->outstanding_tasks);
6804 spin_unlock_irqrestore(hba->host->host_lock, flags);
6805
6806 ufshcd_release(hba);
6807 blk_mq_free_request(req);
6808
6809 return err;
6810 }
6811
6812 /**
6813 * ufshcd_issue_tm_cmd - issues task management commands to controller
6814 * @hba: per adapter instance
6815 * @lun_id: LUN ID to which TM command is sent
6816 * @task_id: task ID to which the TM command is applicable
6817 * @tm_function: task management function opcode
6818 * @tm_response: task management service response return value
6819 *
6820 * Returns non-zero value on error, zero on success.
6821 */
ufshcd_issue_tm_cmd(struct ufs_hba * hba,int lun_id,int task_id,u8 tm_function,u8 * tm_response)6822 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6823 u8 tm_function, u8 *tm_response)
6824 {
6825 struct utp_task_req_desc treq = { { 0 }, };
6826 enum utp_ocs ocs_value;
6827 int err;
6828
6829 /* Configure task request descriptor */
6830 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6831 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6832
6833 /* Configure task request UPIU */
6834 treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6835 cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6836 treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6837
6838 /*
6839 * The host shall provide the same value for LUN field in the basic
6840 * header and for Input Parameter.
6841 */
6842 treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
6843 treq.upiu_req.input_param2 = cpu_to_be32(task_id);
6844
6845 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6846 if (err == -ETIMEDOUT)
6847 return err;
6848
6849 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6850 if (ocs_value != OCS_SUCCESS)
6851 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6852 __func__, ocs_value);
6853 else if (tm_response)
6854 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
6855 MASK_TM_SERVICE_RESP;
6856 return err;
6857 }
6858
6859 /**
6860 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6861 * @hba: per-adapter instance
6862 * @req_upiu: upiu request
6863 * @rsp_upiu: upiu reply
6864 * @desc_buff: pointer to descriptor buffer, NULL if NA
6865 * @buff_len: descriptor size, 0 if NA
6866 * @cmd_type: specifies the type (NOP, Query...)
6867 * @desc_op: descriptor operation
6868 *
6869 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6870 * Therefore, it "rides" the device management infrastructure: uses its tag and
6871 * tasks work queues.
6872 *
6873 * Since there is only one available tag for device management commands,
6874 * the caller is expected to hold the hba->dev_cmd.lock mutex.
6875 */
ufshcd_issue_devman_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,u8 * desc_buff,int * buff_len,enum dev_cmd_type cmd_type,enum query_opcode desc_op)6876 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6877 struct utp_upiu_req *req_upiu,
6878 struct utp_upiu_req *rsp_upiu,
6879 u8 *desc_buff, int *buff_len,
6880 enum dev_cmd_type cmd_type,
6881 enum query_opcode desc_op)
6882 {
6883 DECLARE_COMPLETION_ONSTACK(wait);
6884 const u32 tag = hba->reserved_slot;
6885 struct ufshcd_lrb *lrbp;
6886 int err = 0;
6887 u8 upiu_flags;
6888
6889 /* Protects use of hba->reserved_slot. */
6890 lockdep_assert_held(&hba->dev_cmd.lock);
6891
6892 down_read(&hba->clk_scaling_lock);
6893
6894 lrbp = &hba->lrb[tag];
6895 WARN_ON(lrbp->cmd);
6896 lrbp->cmd = NULL;
6897 lrbp->task_tag = tag;
6898 lrbp->lun = 0;
6899 lrbp->intr_cmd = true;
6900 ufshcd_prepare_lrbp_crypto(NULL, lrbp);
6901 hba->dev_cmd.type = cmd_type;
6902
6903 if (hba->ufs_version <= ufshci_version(1, 1))
6904 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6905 else
6906 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6907
6908 /* update the task tag in the request upiu */
6909 req_upiu->header.dword_0 |= cpu_to_be32(tag);
6910
6911 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6912
6913 /* just copy the upiu request as it is */
6914 memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6915 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6916 /* The Data Segment Area is optional depending upon the query
6917 * function value. for WRITE DESCRIPTOR, the data segment
6918 * follows right after the tsf.
6919 */
6920 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6921 *buff_len = 0;
6922 }
6923
6924 memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6925
6926 hba->dev_cmd.complete = &wait;
6927
6928 ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
6929
6930 ufshcd_send_command(hba, tag);
6931 /*
6932 * ignore the returning value here - ufshcd_check_query_response is
6933 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6934 * read the response directly ignoring all errors.
6935 */
6936 ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6937
6938 /* just copy the upiu response as it is */
6939 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
6940 if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6941 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6942 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6943 MASK_QUERY_DATA_SEG_LEN;
6944
6945 if (*buff_len >= resp_len) {
6946 memcpy(desc_buff, descp, resp_len);
6947 *buff_len = resp_len;
6948 } else {
6949 dev_warn(hba->dev,
6950 "%s: rsp size %d is bigger than buffer size %d",
6951 __func__, resp_len, *buff_len);
6952 *buff_len = 0;
6953 err = -EINVAL;
6954 }
6955 }
6956 ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
6957 (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
6958
6959 up_read(&hba->clk_scaling_lock);
6960 return err;
6961 }
6962
6963 /**
6964 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6965 * @hba: per-adapter instance
6966 * @req_upiu: upiu request
6967 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
6968 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
6969 * @desc_buff: pointer to descriptor buffer, NULL if NA
6970 * @buff_len: descriptor size, 0 if NA
6971 * @desc_op: descriptor operation
6972 *
6973 * Supports UTP Transfer requests (nop and query), and UTP Task
6974 * Management requests.
6975 * It is up to the caller to fill the upiu conent properly, as it will
6976 * be copied without any further input validations.
6977 */
ufshcd_exec_raw_upiu_cmd(struct ufs_hba * hba,struct utp_upiu_req * req_upiu,struct utp_upiu_req * rsp_upiu,int msgcode,u8 * desc_buff,int * buff_len,enum query_opcode desc_op)6978 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6979 struct utp_upiu_req *req_upiu,
6980 struct utp_upiu_req *rsp_upiu,
6981 int msgcode,
6982 u8 *desc_buff, int *buff_len,
6983 enum query_opcode desc_op)
6984 {
6985 int err;
6986 enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
6987 struct utp_task_req_desc treq = { { 0 }, };
6988 enum utp_ocs ocs_value;
6989 u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6990
6991 switch (msgcode) {
6992 case UPIU_TRANSACTION_NOP_OUT:
6993 cmd_type = DEV_CMD_TYPE_NOP;
6994 fallthrough;
6995 case UPIU_TRANSACTION_QUERY_REQ:
6996 ufshcd_hold(hba, false);
6997 mutex_lock(&hba->dev_cmd.lock);
6998 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6999 desc_buff, buff_len,
7000 cmd_type, desc_op);
7001 mutex_unlock(&hba->dev_cmd.lock);
7002 ufshcd_release(hba);
7003
7004 break;
7005 case UPIU_TRANSACTION_TASK_REQ:
7006 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
7007 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
7008
7009 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
7010
7011 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
7012 if (err == -ETIMEDOUT)
7013 break;
7014
7015 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
7016 if (ocs_value != OCS_SUCCESS) {
7017 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
7018 ocs_value);
7019 break;
7020 }
7021
7022 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
7023
7024 break;
7025 default:
7026 err = -EINVAL;
7027
7028 break;
7029 }
7030
7031 return err;
7032 }
7033
7034 /**
7035 * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
7036 * @cmd: SCSI command pointer
7037 *
7038 * Returns SUCCESS/FAILED
7039 */
ufshcd_eh_device_reset_handler(struct scsi_cmnd * cmd)7040 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7041 {
7042 unsigned long flags, pending_reqs = 0, not_cleared = 0;
7043 struct Scsi_Host *host;
7044 struct ufs_hba *hba;
7045 u32 pos;
7046 int err;
7047 u8 resp = 0xF, lun;
7048
7049 host = cmd->device->host;
7050 hba = shost_priv(host);
7051
7052 lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7053 err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
7054 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7055 if (!err)
7056 err = resp;
7057 goto out;
7058 }
7059
7060 /* clear the commands that were pending for corresponding LUN */
7061 spin_lock_irqsave(&hba->outstanding_lock, flags);
7062 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7063 if (hba->lrb[pos].lun == lun)
7064 __set_bit(pos, &pending_reqs);
7065 hba->outstanding_reqs &= ~pending_reqs;
7066 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7067
7068 if (ufshcd_clear_cmds(hba, pending_reqs) < 0) {
7069 spin_lock_irqsave(&hba->outstanding_lock, flags);
7070 not_cleared = pending_reqs &
7071 ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7072 hba->outstanding_reqs |= not_cleared;
7073 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7074
7075 dev_err(hba->dev, "%s: failed to clear requests %#lx\n",
7076 __func__, not_cleared);
7077 }
7078 __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared);
7079
7080 out:
7081 hba->req_abort_count = 0;
7082 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
7083 if (!err) {
7084 err = SUCCESS;
7085 } else {
7086 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7087 err = FAILED;
7088 }
7089 return err;
7090 }
7091
ufshcd_set_req_abort_skip(struct ufs_hba * hba,unsigned long bitmap)7092 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7093 {
7094 struct ufshcd_lrb *lrbp;
7095 int tag;
7096
7097 for_each_set_bit(tag, &bitmap, hba->nutrs) {
7098 lrbp = &hba->lrb[tag];
7099 lrbp->req_abort_skip = true;
7100 }
7101 }
7102
7103 /**
7104 * ufshcd_try_to_abort_task - abort a specific task
7105 * @hba: Pointer to adapter instance
7106 * @tag: Task tag/index to be aborted
7107 *
7108 * Abort the pending command in device by sending UFS_ABORT_TASK task management
7109 * command, and in host controller by clearing the door-bell register. There can
7110 * be race between controller sending the command to the device while abort is
7111 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7112 * really issued and then try to abort it.
7113 *
7114 * Returns zero on success, non-zero on failure
7115 */
ufshcd_try_to_abort_task(struct ufs_hba * hba,int tag)7116 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7117 {
7118 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7119 int err = 0;
7120 int poll_cnt;
7121 u8 resp = 0xF;
7122 u32 reg;
7123
7124 for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7125 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7126 UFS_QUERY_TASK, &resp);
7127 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7128 /* cmd pending in the device */
7129 dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7130 __func__, tag);
7131 break;
7132 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7133 /*
7134 * cmd not pending in the device, check if it is
7135 * in transition.
7136 */
7137 dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
7138 __func__, tag);
7139 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7140 if (reg & (1 << tag)) {
7141 /* sleep for max. 200us to stabilize */
7142 usleep_range(100, 200);
7143 continue;
7144 }
7145 /* command completed already */
7146 dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
7147 __func__, tag);
7148 goto out;
7149 } else {
7150 dev_err(hba->dev,
7151 "%s: no response from device. tag = %d, err %d\n",
7152 __func__, tag, err);
7153 if (!err)
7154 err = resp; /* service response error */
7155 goto out;
7156 }
7157 }
7158
7159 if (!poll_cnt) {
7160 err = -EBUSY;
7161 goto out;
7162 }
7163
7164 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7165 UFS_ABORT_TASK, &resp);
7166 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7167 if (!err) {
7168 err = resp; /* service response error */
7169 dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7170 __func__, tag, err);
7171 }
7172 goto out;
7173 }
7174
7175 err = ufshcd_clear_cmds(hba, 1U << tag);
7176 if (err)
7177 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7178 __func__, tag, err);
7179
7180 out:
7181 return err;
7182 }
7183
7184 /**
7185 * ufshcd_abort - scsi host template eh_abort_handler callback
7186 * @cmd: SCSI command pointer
7187 *
7188 * Returns SUCCESS/FAILED
7189 */
ufshcd_abort(struct scsi_cmnd * cmd)7190 static int ufshcd_abort(struct scsi_cmnd *cmd)
7191 {
7192 struct Scsi_Host *host = cmd->device->host;
7193 struct ufs_hba *hba = shost_priv(host);
7194 int tag = scsi_cmd_to_rq(cmd)->tag;
7195 struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7196 unsigned long flags;
7197 int err = FAILED;
7198 bool outstanding;
7199 u32 reg;
7200
7201 WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
7202
7203 ufshcd_hold(hba, false);
7204 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7205 /* If command is already aborted/completed, return FAILED. */
7206 if (!(test_bit(tag, &hba->outstanding_reqs))) {
7207 dev_err(hba->dev,
7208 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7209 __func__, tag, hba->outstanding_reqs, reg);
7210 goto release;
7211 }
7212
7213 /* Print Transfer Request of aborted task */
7214 dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7215
7216 /*
7217 * Print detailed info about aborted request.
7218 * As more than one request might get aborted at the same time,
7219 * print full information only for the first aborted request in order
7220 * to reduce repeated printouts. For other aborted requests only print
7221 * basic details.
7222 */
7223 scsi_print_command(cmd);
7224 if (!hba->req_abort_count) {
7225 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7226 ufshcd_print_evt_hist(hba);
7227 ufshcd_print_host_state(hba);
7228 ufshcd_print_pwr_info(hba);
7229 ufshcd_print_trs(hba, 1 << tag, true);
7230 } else {
7231 ufshcd_print_trs(hba, 1 << tag, false);
7232 }
7233 hba->req_abort_count++;
7234
7235 if (!(reg & (1 << tag))) {
7236 dev_err(hba->dev,
7237 "%s: cmd was completed, but without a notifying intr, tag = %d",
7238 __func__, tag);
7239 __ufshcd_transfer_req_compl(hba, 1UL << tag);
7240 goto release;
7241 }
7242
7243 /*
7244 * Task abort to the device W-LUN is illegal. When this command
7245 * will fail, due to spec violation, scsi err handling next step
7246 * will be to send LU reset which, again, is a spec violation.
7247 * To avoid these unnecessary/illegal steps, first we clean up
7248 * the lrb taken by this cmd and re-set it in outstanding_reqs,
7249 * then queue the eh_work and bail.
7250 */
7251 if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7252 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7253
7254 spin_lock_irqsave(host->host_lock, flags);
7255 hba->force_reset = true;
7256 ufshcd_schedule_eh_work(hba);
7257 spin_unlock_irqrestore(host->host_lock, flags);
7258 goto release;
7259 }
7260
7261 /* Skip task abort in case previous aborts failed and report failure */
7262 if (lrbp->req_abort_skip) {
7263 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7264 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7265 goto release;
7266 }
7267
7268 err = ufshcd_try_to_abort_task(hba, tag);
7269 if (err) {
7270 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7271 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7272 err = FAILED;
7273 goto release;
7274 }
7275
7276 /*
7277 * Clear the corresponding bit from outstanding_reqs since the command
7278 * has been aborted successfully.
7279 */
7280 spin_lock_irqsave(&hba->outstanding_lock, flags);
7281 outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7282 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7283
7284 if (outstanding)
7285 ufshcd_release_scsi_cmd(hba, lrbp);
7286
7287 err = SUCCESS;
7288
7289 release:
7290 /* Matches the ufshcd_hold() call at the start of this function. */
7291 ufshcd_release(hba);
7292 return err;
7293 }
7294
7295 /**
7296 * ufshcd_host_reset_and_restore - reset and restore host controller
7297 * @hba: per-adapter instance
7298 *
7299 * Note that host controller reset may issue DME_RESET to
7300 * local and remote (device) Uni-Pro stack and the attributes
7301 * are reset to default state.
7302 *
7303 * Returns zero on success, non-zero on failure
7304 */
ufshcd_host_reset_and_restore(struct ufs_hba * hba)7305 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7306 {
7307 int err;
7308
7309 /*
7310 * Stop the host controller and complete the requests
7311 * cleared by h/w
7312 */
7313 ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET);
7314 ufshcd_hba_stop(hba);
7315 hba->silence_err_logs = true;
7316 ufshcd_complete_requests(hba);
7317 hba->silence_err_logs = false;
7318
7319 /* scale up clocks to max frequency before full reinitialization */
7320 ufshcd_scale_clks(hba, true);
7321
7322 err = ufshcd_hba_enable(hba);
7323
7324 /* Establish the link again and restore the device */
7325 if (!err)
7326 err = ufshcd_probe_hba(hba, false);
7327
7328 if (err)
7329 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7330 ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7331 return err;
7332 }
7333
7334 /**
7335 * ufshcd_reset_and_restore - reset and re-initialize host/device
7336 * @hba: per-adapter instance
7337 *
7338 * Reset and recover device, host and re-establish link. This
7339 * is helpful to recover the communication in fatal error conditions.
7340 *
7341 * Returns zero on success, non-zero on failure
7342 */
ufshcd_reset_and_restore(struct ufs_hba * hba)7343 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7344 {
7345 u32 saved_err = 0;
7346 u32 saved_uic_err = 0;
7347 int err = 0;
7348 unsigned long flags;
7349 int retries = MAX_HOST_RESET_RETRIES;
7350
7351 spin_lock_irqsave(hba->host->host_lock, flags);
7352 do {
7353 /*
7354 * This is a fresh start, cache and clear saved error first,
7355 * in case new error generated during reset and restore.
7356 */
7357 saved_err |= hba->saved_err;
7358 saved_uic_err |= hba->saved_uic_err;
7359 hba->saved_err = 0;
7360 hba->saved_uic_err = 0;
7361 hba->force_reset = false;
7362 hba->ufshcd_state = UFSHCD_STATE_RESET;
7363 spin_unlock_irqrestore(hba->host->host_lock, flags);
7364
7365 /* Reset the attached device */
7366 ufshcd_device_reset(hba);
7367
7368 err = ufshcd_host_reset_and_restore(hba);
7369
7370 spin_lock_irqsave(hba->host->host_lock, flags);
7371 if (err)
7372 continue;
7373 /* Do not exit unless operational or dead */
7374 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7375 hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7376 hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7377 err = -EAGAIN;
7378 } while (err && --retries);
7379
7380 /*
7381 * Inform scsi mid-layer that we did reset and allow to handle
7382 * Unit Attention properly.
7383 */
7384 scsi_report_bus_reset(hba->host, 0);
7385 if (err) {
7386 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7387 hba->saved_err |= saved_err;
7388 hba->saved_uic_err |= saved_uic_err;
7389 }
7390 spin_unlock_irqrestore(hba->host->host_lock, flags);
7391
7392 return err;
7393 }
7394
7395 /**
7396 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7397 * @cmd: SCSI command pointer
7398 *
7399 * Returns SUCCESS/FAILED
7400 */
ufshcd_eh_host_reset_handler(struct scsi_cmnd * cmd)7401 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7402 {
7403 int err = SUCCESS;
7404 unsigned long flags;
7405 struct ufs_hba *hba;
7406
7407 hba = shost_priv(cmd->device->host);
7408
7409 spin_lock_irqsave(hba->host->host_lock, flags);
7410 hba->force_reset = true;
7411 ufshcd_schedule_eh_work(hba);
7412 dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7413 spin_unlock_irqrestore(hba->host->host_lock, flags);
7414
7415 flush_work(&hba->eh_work);
7416
7417 spin_lock_irqsave(hba->host->host_lock, flags);
7418 if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7419 err = FAILED;
7420 spin_unlock_irqrestore(hba->host->host_lock, flags);
7421
7422 return err;
7423 }
7424
7425 /**
7426 * ufshcd_get_max_icc_level - calculate the ICC level
7427 * @sup_curr_uA: max. current supported by the regulator
7428 * @start_scan: row at the desc table to start scan from
7429 * @buff: power descriptor buffer
7430 *
7431 * Returns calculated max ICC level for specific regulator
7432 */
ufshcd_get_max_icc_level(int sup_curr_uA,u32 start_scan,const char * buff)7433 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan,
7434 const char *buff)
7435 {
7436 int i;
7437 int curr_uA;
7438 u16 data;
7439 u16 unit;
7440
7441 for (i = start_scan; i >= 0; i--) {
7442 data = get_unaligned_be16(&buff[2 * i]);
7443 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7444 ATTR_ICC_LVL_UNIT_OFFSET;
7445 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7446 switch (unit) {
7447 case UFSHCD_NANO_AMP:
7448 curr_uA = curr_uA / 1000;
7449 break;
7450 case UFSHCD_MILI_AMP:
7451 curr_uA = curr_uA * 1000;
7452 break;
7453 case UFSHCD_AMP:
7454 curr_uA = curr_uA * 1000 * 1000;
7455 break;
7456 case UFSHCD_MICRO_AMP:
7457 default:
7458 break;
7459 }
7460 if (sup_curr_uA >= curr_uA)
7461 break;
7462 }
7463 if (i < 0) {
7464 i = 0;
7465 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7466 }
7467
7468 return (u32)i;
7469 }
7470
7471 /**
7472 * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7473 * In case regulators are not initialized we'll return 0
7474 * @hba: per-adapter instance
7475 * @desc_buf: power descriptor buffer to extract ICC levels from.
7476 * @len: length of desc_buff
7477 *
7478 * Returns calculated ICC level
7479 */
ufshcd_find_max_sup_active_icc_level(struct ufs_hba * hba,const u8 * desc_buf,int len)7480 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7481 const u8 *desc_buf, int len)
7482 {
7483 u32 icc_level = 0;
7484
7485 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7486 !hba->vreg_info.vccq2) {
7487 /*
7488 * Using dev_dbg to avoid messages during runtime PM to avoid
7489 * never-ending cycles of messages written back to storage by
7490 * user space causing runtime resume, causing more messages and
7491 * so on.
7492 */
7493 dev_dbg(hba->dev,
7494 "%s: Regulator capability was not set, actvIccLevel=%d",
7495 __func__, icc_level);
7496 goto out;
7497 }
7498
7499 if (hba->vreg_info.vcc->max_uA)
7500 icc_level = ufshcd_get_max_icc_level(
7501 hba->vreg_info.vcc->max_uA,
7502 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7503 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7504
7505 if (hba->vreg_info.vccq->max_uA)
7506 icc_level = ufshcd_get_max_icc_level(
7507 hba->vreg_info.vccq->max_uA,
7508 icc_level,
7509 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7510
7511 if (hba->vreg_info.vccq2->max_uA)
7512 icc_level = ufshcd_get_max_icc_level(
7513 hba->vreg_info.vccq2->max_uA,
7514 icc_level,
7515 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7516 out:
7517 return icc_level;
7518 }
7519
ufshcd_set_active_icc_lvl(struct ufs_hba * hba)7520 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7521 {
7522 int ret;
7523 int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
7524 u8 *desc_buf;
7525 u32 icc_level;
7526
7527 desc_buf = kmalloc(buff_len, GFP_KERNEL);
7528 if (!desc_buf)
7529 return;
7530
7531 ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7532 desc_buf, buff_len);
7533 if (ret) {
7534 dev_err(hba->dev,
7535 "%s: Failed reading power descriptor.len = %d ret = %d",
7536 __func__, buff_len, ret);
7537 goto out;
7538 }
7539
7540 icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7541 buff_len);
7542 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7543
7544 ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7545 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7546
7547 if (ret)
7548 dev_err(hba->dev,
7549 "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7550 __func__, icc_level, ret);
7551
7552 out:
7553 kfree(desc_buf);
7554 }
7555
ufshcd_blk_pm_runtime_init(struct scsi_device * sdev)7556 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7557 {
7558 scsi_autopm_get_device(sdev);
7559 blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7560 if (sdev->rpm_autosuspend)
7561 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7562 RPM_AUTOSUSPEND_DELAY_MS);
7563 scsi_autopm_put_device(sdev);
7564 }
7565
7566 /**
7567 * ufshcd_scsi_add_wlus - Adds required W-LUs
7568 * @hba: per-adapter instance
7569 *
7570 * UFS device specification requires the UFS devices to support 4 well known
7571 * logical units:
7572 * "REPORT_LUNS" (address: 01h)
7573 * "UFS Device" (address: 50h)
7574 * "RPMB" (address: 44h)
7575 * "BOOT" (address: 30h)
7576 * UFS device's power management needs to be controlled by "POWER CONDITION"
7577 * field of SSU (START STOP UNIT) command. But this "power condition" field
7578 * will take effect only when its sent to "UFS device" well known logical unit
7579 * hence we require the scsi_device instance to represent this logical unit in
7580 * order for the UFS host driver to send the SSU command for power management.
7581 *
7582 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7583 * Block) LU so user space process can control this LU. User space may also
7584 * want to have access to BOOT LU.
7585 *
7586 * This function adds scsi device instances for each of all well known LUs
7587 * (except "REPORT LUNS" LU).
7588 *
7589 * Returns zero on success (all required W-LUs are added successfully),
7590 * non-zero error value on failure (if failed to add any of the required W-LU).
7591 */
ufshcd_scsi_add_wlus(struct ufs_hba * hba)7592 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7593 {
7594 int ret = 0;
7595 struct scsi_device *sdev_boot, *sdev_rpmb;
7596
7597 hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
7598 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7599 if (IS_ERR(hba->ufs_device_wlun)) {
7600 ret = PTR_ERR(hba->ufs_device_wlun);
7601 hba->ufs_device_wlun = NULL;
7602 goto out;
7603 }
7604 scsi_device_put(hba->ufs_device_wlun);
7605
7606 sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7607 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7608 if (IS_ERR(sdev_rpmb)) {
7609 ret = PTR_ERR(sdev_rpmb);
7610 goto remove_ufs_device_wlun;
7611 }
7612 ufshcd_blk_pm_runtime_init(sdev_rpmb);
7613 scsi_device_put(sdev_rpmb);
7614
7615 sdev_boot = __scsi_add_device(hba->host, 0, 0,
7616 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
7617 if (IS_ERR(sdev_boot)) {
7618 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
7619 } else {
7620 ufshcd_blk_pm_runtime_init(sdev_boot);
7621 scsi_device_put(sdev_boot);
7622 }
7623 goto out;
7624
7625 remove_ufs_device_wlun:
7626 scsi_remove_device(hba->ufs_device_wlun);
7627 out:
7628 return ret;
7629 }
7630
ufshcd_wb_probe(struct ufs_hba * hba,const u8 * desc_buf)7631 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf)
7632 {
7633 struct ufs_dev_info *dev_info = &hba->dev_info;
7634 u8 lun;
7635 u32 d_lu_wb_buf_alloc;
7636 u32 ext_ufs_feature;
7637
7638 if (!ufshcd_is_wb_allowed(hba))
7639 return;
7640
7641 /*
7642 * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7643 * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7644 * enabled
7645 */
7646 if (!(dev_info->wspecversion >= 0x310 ||
7647 dev_info->wspecversion == 0x220 ||
7648 (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7649 goto wb_disabled;
7650
7651 if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7652 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
7653 goto wb_disabled;
7654
7655 ext_ufs_feature = get_unaligned_be32(desc_buf +
7656 DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7657
7658 if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
7659 goto wb_disabled;
7660
7661 /*
7662 * WB may be supported but not configured while provisioning. The spec
7663 * says, in dedicated wb buffer mode, a max of 1 lun would have wb
7664 * buffer configured.
7665 */
7666 dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7667
7668 dev_info->b_presrv_uspc_en =
7669 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7670
7671 if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
7672 if (!get_unaligned_be32(desc_buf +
7673 DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
7674 goto wb_disabled;
7675 } else {
7676 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7677 d_lu_wb_buf_alloc = 0;
7678 ufshcd_read_unit_desc_param(hba,
7679 lun,
7680 UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7681 (u8 *)&d_lu_wb_buf_alloc,
7682 sizeof(d_lu_wb_buf_alloc));
7683 if (d_lu_wb_buf_alloc) {
7684 dev_info->wb_dedicated_lu = lun;
7685 break;
7686 }
7687 }
7688
7689 if (!d_lu_wb_buf_alloc)
7690 goto wb_disabled;
7691 }
7692
7693 if (!ufshcd_is_wb_buf_lifetime_available(hba))
7694 goto wb_disabled;
7695
7696 return;
7697
7698 wb_disabled:
7699 hba->caps &= ~UFSHCD_CAP_WB_EN;
7700 }
7701
ufshcd_temp_notif_probe(struct ufs_hba * hba,const u8 * desc_buf)7702 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf)
7703 {
7704 struct ufs_dev_info *dev_info = &hba->dev_info;
7705 u32 ext_ufs_feature;
7706 u8 mask = 0;
7707
7708 if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
7709 return;
7710
7711 ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7712
7713 if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
7714 mask |= MASK_EE_TOO_LOW_TEMP;
7715
7716 if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
7717 mask |= MASK_EE_TOO_HIGH_TEMP;
7718
7719 if (mask) {
7720 ufshcd_enable_ee(hba, mask);
7721 ufs_hwmon_probe(hba, mask);
7722 }
7723 }
7724
ufshcd_fixup_dev_quirks(struct ufs_hba * hba,const struct ufs_dev_quirk * fixups)7725 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
7726 const struct ufs_dev_quirk *fixups)
7727 {
7728 const struct ufs_dev_quirk *f;
7729 struct ufs_dev_info *dev_info = &hba->dev_info;
7730
7731 if (!fixups)
7732 return;
7733
7734 for (f = fixups; f->quirk; f++) {
7735 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7736 f->wmanufacturerid == UFS_ANY_VENDOR) &&
7737 ((dev_info->model &&
7738 STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7739 !strcmp(f->model, UFS_ANY_MODEL)))
7740 hba->dev_quirks |= f->quirk;
7741 }
7742 }
7743 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
7744
ufs_fixup_device_setup(struct ufs_hba * hba)7745 static void ufs_fixup_device_setup(struct ufs_hba *hba)
7746 {
7747 /* fix by general quirk table */
7748 ufshcd_fixup_dev_quirks(hba, ufs_fixups);
7749
7750 /* allow vendors to fix quirks */
7751 ufshcd_vops_fixup_dev_quirks(hba);
7752 }
7753
ufs_get_device_desc(struct ufs_hba * hba)7754 static int ufs_get_device_desc(struct ufs_hba *hba)
7755 {
7756 int err;
7757 u8 model_index;
7758 u8 b_ufs_feature_sup;
7759 u8 *desc_buf;
7760 struct ufs_dev_info *dev_info = &hba->dev_info;
7761
7762 desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7763 if (!desc_buf) {
7764 err = -ENOMEM;
7765 goto out;
7766 }
7767
7768 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7769 hba->desc_size[QUERY_DESC_IDN_DEVICE]);
7770 if (err) {
7771 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7772 __func__, err);
7773 goto out;
7774 }
7775
7776 /*
7777 * getting vendor (manufacturerID) and Bank Index in big endian
7778 * format
7779 */
7780 dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
7781 desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7782
7783 /* getting Specification Version in big endian format */
7784 dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7785 desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7786 b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT];
7787
7788 model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
7789
7790 if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION &&
7791 (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) {
7792 bool hpb_en = false;
7793
7794 ufshpb_get_dev_info(hba, desc_buf);
7795
7796 if (!ufshpb_is_legacy(hba))
7797 err = ufshcd_query_flag_retry(hba,
7798 UPIU_QUERY_OPCODE_READ_FLAG,
7799 QUERY_FLAG_IDN_HPB_EN, 0,
7800 &hpb_en);
7801
7802 if (ufshpb_is_legacy(hba) || (!err && hpb_en))
7803 dev_info->hpb_enabled = true;
7804 }
7805
7806 err = ufshcd_read_string_desc(hba, model_index,
7807 &dev_info->model, SD_ASCII_STD);
7808 if (err < 0) {
7809 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7810 __func__, err);
7811 goto out;
7812 }
7813
7814 hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
7815 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
7816
7817 ufs_fixup_device_setup(hba);
7818
7819 ufshcd_wb_probe(hba, desc_buf);
7820
7821 ufshcd_temp_notif_probe(hba, desc_buf);
7822
7823 /*
7824 * ufshcd_read_string_desc returns size of the string
7825 * reset the error value
7826 */
7827 err = 0;
7828
7829 out:
7830 kfree(desc_buf);
7831 return err;
7832 }
7833
ufs_put_device_desc(struct ufs_hba * hba)7834 static void ufs_put_device_desc(struct ufs_hba *hba)
7835 {
7836 struct ufs_dev_info *dev_info = &hba->dev_info;
7837
7838 kfree(dev_info->model);
7839 dev_info->model = NULL;
7840 }
7841
7842 /**
7843 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7844 * @hba: per-adapter instance
7845 *
7846 * PA_TActivate parameter can be tuned manually if UniPro version is less than
7847 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7848 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7849 * the hibern8 exit latency.
7850 *
7851 * Returns zero on success, non-zero error value on failure.
7852 */
ufshcd_tune_pa_tactivate(struct ufs_hba * hba)7853 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7854 {
7855 int ret = 0;
7856 u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7857
7858 ret = ufshcd_dme_peer_get(hba,
7859 UIC_ARG_MIB_SEL(
7860 RX_MIN_ACTIVATETIME_CAPABILITY,
7861 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7862 &peer_rx_min_activatetime);
7863 if (ret)
7864 goto out;
7865
7866 /* make sure proper unit conversion is applied */
7867 tuned_pa_tactivate =
7868 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7869 / PA_TACTIVATE_TIME_UNIT_US);
7870 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7871 tuned_pa_tactivate);
7872
7873 out:
7874 return ret;
7875 }
7876
7877 /**
7878 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7879 * @hba: per-adapter instance
7880 *
7881 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7882 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7883 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7884 * This optimal value can help reduce the hibern8 exit latency.
7885 *
7886 * Returns zero on success, non-zero error value on failure.
7887 */
ufshcd_tune_pa_hibern8time(struct ufs_hba * hba)7888 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7889 {
7890 int ret = 0;
7891 u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7892 u32 max_hibern8_time, tuned_pa_hibern8time;
7893
7894 ret = ufshcd_dme_get(hba,
7895 UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7896 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7897 &local_tx_hibern8_time_cap);
7898 if (ret)
7899 goto out;
7900
7901 ret = ufshcd_dme_peer_get(hba,
7902 UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7903 UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7904 &peer_rx_hibern8_time_cap);
7905 if (ret)
7906 goto out;
7907
7908 max_hibern8_time = max(local_tx_hibern8_time_cap,
7909 peer_rx_hibern8_time_cap);
7910 /* make sure proper unit conversion is applied */
7911 tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7912 / PA_HIBERN8_TIME_UNIT_US);
7913 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7914 tuned_pa_hibern8time);
7915 out:
7916 return ret;
7917 }
7918
7919 /**
7920 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7921 * less than device PA_TACTIVATE time.
7922 * @hba: per-adapter instance
7923 *
7924 * Some UFS devices require host PA_TACTIVATE to be lower than device
7925 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7926 * for such devices.
7927 *
7928 * Returns zero on success, non-zero error value on failure.
7929 */
ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba * hba)7930 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7931 {
7932 int ret = 0;
7933 u32 granularity, peer_granularity;
7934 u32 pa_tactivate, peer_pa_tactivate;
7935 u32 pa_tactivate_us, peer_pa_tactivate_us;
7936 static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7937
7938 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7939 &granularity);
7940 if (ret)
7941 goto out;
7942
7943 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7944 &peer_granularity);
7945 if (ret)
7946 goto out;
7947
7948 if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7949 (granularity > PA_GRANULARITY_MAX_VAL)) {
7950 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7951 __func__, granularity);
7952 return -EINVAL;
7953 }
7954
7955 if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7956 (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7957 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7958 __func__, peer_granularity);
7959 return -EINVAL;
7960 }
7961
7962 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7963 if (ret)
7964 goto out;
7965
7966 ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7967 &peer_pa_tactivate);
7968 if (ret)
7969 goto out;
7970
7971 pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7972 peer_pa_tactivate_us = peer_pa_tactivate *
7973 gran_to_us_table[peer_granularity - 1];
7974
7975 if (pa_tactivate_us >= peer_pa_tactivate_us) {
7976 u32 new_peer_pa_tactivate;
7977
7978 new_peer_pa_tactivate = pa_tactivate_us /
7979 gran_to_us_table[peer_granularity - 1];
7980 new_peer_pa_tactivate++;
7981 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7982 new_peer_pa_tactivate);
7983 }
7984
7985 out:
7986 return ret;
7987 }
7988
ufshcd_tune_unipro_params(struct ufs_hba * hba)7989 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
7990 {
7991 if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7992 ufshcd_tune_pa_tactivate(hba);
7993 ufshcd_tune_pa_hibern8time(hba);
7994 }
7995
7996 ufshcd_vops_apply_dev_quirks(hba);
7997
7998 if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7999 /* set 1ms timeout for PA_TACTIVATE */
8000 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
8001
8002 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
8003 ufshcd_quirk_tune_host_pa_tactivate(hba);
8004 }
8005
ufshcd_clear_dbg_ufs_stats(struct ufs_hba * hba)8006 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
8007 {
8008 hba->ufs_stats.hibern8_exit_cnt = 0;
8009 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
8010 hba->req_abort_count = 0;
8011 }
8012
ufshcd_device_geo_params_init(struct ufs_hba * hba)8013 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
8014 {
8015 int err;
8016 size_t buff_len;
8017 u8 *desc_buf;
8018
8019 buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
8020 desc_buf = kmalloc(buff_len, GFP_KERNEL);
8021 if (!desc_buf) {
8022 err = -ENOMEM;
8023 goto out;
8024 }
8025
8026 err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
8027 desc_buf, buff_len);
8028 if (err) {
8029 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
8030 __func__, err);
8031 goto out;
8032 }
8033
8034 if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
8035 hba->dev_info.max_lu_supported = 32;
8036 else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
8037 hba->dev_info.max_lu_supported = 8;
8038
8039 if (hba->desc_size[QUERY_DESC_IDN_GEOMETRY] >=
8040 GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS)
8041 ufshpb_get_geo_info(hba, desc_buf);
8042
8043 out:
8044 kfree(desc_buf);
8045 return err;
8046 }
8047
8048 struct ufs_ref_clk {
8049 unsigned long freq_hz;
8050 enum ufs_ref_clk_freq val;
8051 };
8052
8053 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = {
8054 {19200000, REF_CLK_FREQ_19_2_MHZ},
8055 {26000000, REF_CLK_FREQ_26_MHZ},
8056 {38400000, REF_CLK_FREQ_38_4_MHZ},
8057 {52000000, REF_CLK_FREQ_52_MHZ},
8058 {0, REF_CLK_FREQ_INVAL},
8059 };
8060
8061 static enum ufs_ref_clk_freq
ufs_get_bref_clk_from_hz(unsigned long freq)8062 ufs_get_bref_clk_from_hz(unsigned long freq)
8063 {
8064 int i;
8065
8066 for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
8067 if (ufs_ref_clk_freqs[i].freq_hz == freq)
8068 return ufs_ref_clk_freqs[i].val;
8069
8070 return REF_CLK_FREQ_INVAL;
8071 }
8072
ufshcd_parse_dev_ref_clk_freq(struct ufs_hba * hba,struct clk * refclk)8073 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
8074 {
8075 unsigned long freq;
8076
8077 freq = clk_get_rate(refclk);
8078
8079 hba->dev_ref_clk_freq =
8080 ufs_get_bref_clk_from_hz(freq);
8081
8082 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8083 dev_err(hba->dev,
8084 "invalid ref_clk setting = %ld\n", freq);
8085 }
8086
ufshcd_set_dev_ref_clk(struct ufs_hba * hba)8087 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8088 {
8089 int err;
8090 u32 ref_clk;
8091 u32 freq = hba->dev_ref_clk_freq;
8092
8093 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8094 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8095
8096 if (err) {
8097 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8098 err);
8099 goto out;
8100 }
8101
8102 if (ref_clk == freq)
8103 goto out; /* nothing to update */
8104
8105 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8106 QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8107
8108 if (err) {
8109 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8110 ufs_ref_clk_freqs[freq].freq_hz);
8111 goto out;
8112 }
8113
8114 dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8115 ufs_ref_clk_freqs[freq].freq_hz);
8116
8117 out:
8118 return err;
8119 }
8120
ufshcd_device_params_init(struct ufs_hba * hba)8121 static int ufshcd_device_params_init(struct ufs_hba *hba)
8122 {
8123 bool flag;
8124 int ret, i;
8125
8126 /* Init device descriptor sizes */
8127 for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
8128 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
8129
8130 /* Init UFS geometry descriptor related parameters */
8131 ret = ufshcd_device_geo_params_init(hba);
8132 if (ret)
8133 goto out;
8134
8135 /* Check and apply UFS device quirks */
8136 ret = ufs_get_device_desc(hba);
8137 if (ret) {
8138 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8139 __func__, ret);
8140 goto out;
8141 }
8142
8143 ufshcd_get_ref_clk_gating_wait(hba);
8144
8145 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
8146 QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
8147 hba->dev_info.f_power_on_wp_en = flag;
8148
8149 /* Probe maximum power mode co-supported by both UFS host and device */
8150 if (ufshcd_get_max_pwr_mode(hba))
8151 dev_err(hba->dev,
8152 "%s: Failed getting max supported power mode\n",
8153 __func__);
8154 out:
8155 return ret;
8156 }
8157
8158 /**
8159 * ufshcd_add_lus - probe and add UFS logical units
8160 * @hba: per-adapter instance
8161 */
ufshcd_add_lus(struct ufs_hba * hba)8162 static int ufshcd_add_lus(struct ufs_hba *hba)
8163 {
8164 int ret;
8165
8166 /* Add required well known logical units to scsi mid layer */
8167 ret = ufshcd_scsi_add_wlus(hba);
8168 if (ret)
8169 goto out;
8170
8171 /* Initialize devfreq after UFS device is detected */
8172 if (ufshcd_is_clkscaling_supported(hba)) {
8173 memcpy(&hba->clk_scaling.saved_pwr_info.info,
8174 &hba->pwr_info,
8175 sizeof(struct ufs_pa_layer_attr));
8176 hba->clk_scaling.saved_pwr_info.is_valid = true;
8177 hba->clk_scaling.is_allowed = true;
8178
8179 ret = ufshcd_devfreq_init(hba);
8180 if (ret)
8181 goto out;
8182
8183 hba->clk_scaling.is_enabled = true;
8184 ufshcd_init_clk_scaling_sysfs(hba);
8185 }
8186
8187 ufs_bsg_probe(hba);
8188 ufshpb_init(hba);
8189 scsi_scan_host(hba->host);
8190 pm_runtime_put_sync(hba->dev);
8191
8192 out:
8193 return ret;
8194 }
8195
8196 /**
8197 * ufshcd_probe_hba - probe hba to detect device and initialize it
8198 * @hba: per-adapter instance
8199 * @init_dev_params: whether or not to call ufshcd_device_params_init().
8200 *
8201 * Execute link-startup and verify device initialization
8202 */
ufshcd_probe_hba(struct ufs_hba * hba,bool init_dev_params)8203 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
8204 {
8205 int ret;
8206 unsigned long flags;
8207 ktime_t start = ktime_get();
8208
8209 hba->ufshcd_state = UFSHCD_STATE_RESET;
8210
8211 ret = ufshcd_link_startup(hba);
8212 if (ret)
8213 goto out;
8214
8215 if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8216 goto out;
8217
8218 /* Debug counters initialization */
8219 ufshcd_clear_dbg_ufs_stats(hba);
8220
8221 /* UniPro link is active now */
8222 ufshcd_set_link_active(hba);
8223
8224 /* Verify device initialization by sending NOP OUT UPIU */
8225 ret = ufshcd_verify_dev_init(hba);
8226 if (ret)
8227 goto out;
8228
8229 /* Initiate UFS initialization, and waiting until completion */
8230 ret = ufshcd_complete_dev_init(hba);
8231 if (ret)
8232 goto out;
8233
8234 /*
8235 * Initialize UFS device parameters used by driver, these
8236 * parameters are associated with UFS descriptors.
8237 */
8238 if (init_dev_params) {
8239 ret = ufshcd_device_params_init(hba);
8240 if (ret)
8241 goto out;
8242 }
8243
8244 ufshcd_tune_unipro_params(hba);
8245
8246 /* UFS device is also active now */
8247 ufshcd_set_ufs_dev_active(hba);
8248 ufshcd_force_reset_auto_bkops(hba);
8249
8250 /* Gear up to HS gear if supported */
8251 if (hba->max_pwr_info.is_valid) {
8252 /*
8253 * Set the right value to bRefClkFreq before attempting to
8254 * switch to HS gears.
8255 */
8256 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8257 ufshcd_set_dev_ref_clk(hba);
8258 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8259 if (ret) {
8260 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8261 __func__, ret);
8262 goto out;
8263 }
8264 ufshcd_print_pwr_info(hba);
8265 }
8266
8267 /*
8268 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8269 * and for removable UFS card as well, hence always set the parameter.
8270 * Note: Error handler may issue the device reset hence resetting
8271 * bActiveICCLevel as well so it is always safe to set this here.
8272 */
8273 ufshcd_set_active_icc_lvl(hba);
8274
8275 /* Enable UFS Write Booster if supported */
8276 ufshcd_configure_wb(hba);
8277
8278 if (hba->ee_usr_mask)
8279 ufshcd_write_ee_control(hba);
8280 /* Enable Auto-Hibernate if configured */
8281 ufshcd_auto_hibern8_enable(hba);
8282
8283 ufshpb_toggle_state(hba, HPB_RESET, HPB_PRESENT);
8284 out:
8285 spin_lock_irqsave(hba->host->host_lock, flags);
8286 if (ret)
8287 hba->ufshcd_state = UFSHCD_STATE_ERROR;
8288 else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8289 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8290 spin_unlock_irqrestore(hba->host->host_lock, flags);
8291
8292 trace_ufshcd_init(dev_name(hba->dev), ret,
8293 ktime_to_us(ktime_sub(ktime_get(), start)),
8294 hba->curr_dev_pwr_mode, hba->uic_link_state);
8295 return ret;
8296 }
8297
8298 /**
8299 * ufshcd_async_scan - asynchronous execution for probing hba
8300 * @data: data pointer to pass to this function
8301 * @cookie: cookie data
8302 */
ufshcd_async_scan(void * data,async_cookie_t cookie)8303 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8304 {
8305 struct ufs_hba *hba = (struct ufs_hba *)data;
8306 int ret;
8307
8308 down(&hba->host_sem);
8309 /* Initialize hba, detect and initialize UFS device */
8310 ret = ufshcd_probe_hba(hba, true);
8311 up(&hba->host_sem);
8312 if (ret)
8313 goto out;
8314
8315 /* Probe and add UFS logical units */
8316 ret = ufshcd_add_lus(hba);
8317 out:
8318 /*
8319 * If we failed to initialize the device or the device is not
8320 * present, turn off the power/clocks etc.
8321 */
8322 if (ret) {
8323 pm_runtime_put_sync(hba->dev);
8324 ufshcd_hba_exit(hba);
8325 }
8326 }
8327
8328 static const struct attribute_group *ufshcd_driver_groups[] = {
8329 &ufs_sysfs_unit_descriptor_group,
8330 &ufs_sysfs_lun_attributes_group,
8331 #ifdef CONFIG_SCSI_UFS_HPB
8332 &ufs_sysfs_hpb_stat_group,
8333 &ufs_sysfs_hpb_param_group,
8334 #endif
8335 NULL,
8336 };
8337
8338 static struct ufs_hba_variant_params ufs_hba_vps = {
8339 .hba_enable_delay_us = 1000,
8340 .wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(40),
8341 .devfreq_profile.polling_ms = 100,
8342 .devfreq_profile.target = ufshcd_devfreq_target,
8343 .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8344 .ondemand_data.upthreshold = 70,
8345 .ondemand_data.downdifferential = 5,
8346 };
8347
8348 static struct scsi_host_template ufshcd_driver_template = {
8349 .module = THIS_MODULE,
8350 .name = UFSHCD,
8351 .proc_name = UFSHCD,
8352 .map_queues = ufshcd_map_queues,
8353 .queuecommand = ufshcd_queuecommand,
8354 .mq_poll = ufshcd_poll,
8355 .slave_alloc = ufshcd_slave_alloc,
8356 .slave_configure = ufshcd_slave_configure,
8357 .slave_destroy = ufshcd_slave_destroy,
8358 .change_queue_depth = ufshcd_change_queue_depth,
8359 .eh_abort_handler = ufshcd_abort,
8360 .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8361 .eh_host_reset_handler = ufshcd_eh_host_reset_handler,
8362 .this_id = -1,
8363 .sg_tablesize = SG_ALL,
8364 .cmd_per_lun = UFSHCD_CMD_PER_LUN,
8365 .can_queue = UFSHCD_CAN_QUEUE,
8366 .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX,
8367 .max_sectors = (1 << 20) / SECTOR_SIZE, /* 1 MiB */
8368 .max_host_blocked = 1,
8369 .track_queue_depth = 1,
8370 .sdev_groups = ufshcd_driver_groups,
8371 .dma_boundary = PAGE_SIZE - 1,
8372 .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS,
8373 };
8374
ufshcd_config_vreg_load(struct device * dev,struct ufs_vreg * vreg,int ua)8375 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8376 int ua)
8377 {
8378 int ret;
8379
8380 if (!vreg)
8381 return 0;
8382
8383 /*
8384 * "set_load" operation shall be required on those regulators
8385 * which specifically configured current limitation. Otherwise
8386 * zero max_uA may cause unexpected behavior when regulator is
8387 * enabled or set as high power mode.
8388 */
8389 if (!vreg->max_uA)
8390 return 0;
8391
8392 ret = regulator_set_load(vreg->reg, ua);
8393 if (ret < 0) {
8394 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
8395 __func__, vreg->name, ua, ret);
8396 }
8397
8398 return ret;
8399 }
8400
ufshcd_config_vreg_lpm(struct ufs_hba * hba,struct ufs_vreg * vreg)8401 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
8402 struct ufs_vreg *vreg)
8403 {
8404 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
8405 }
8406
ufshcd_config_vreg_hpm(struct ufs_hba * hba,struct ufs_vreg * vreg)8407 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
8408 struct ufs_vreg *vreg)
8409 {
8410 if (!vreg)
8411 return 0;
8412
8413 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
8414 }
8415
ufshcd_config_vreg(struct device * dev,struct ufs_vreg * vreg,bool on)8416 static int ufshcd_config_vreg(struct device *dev,
8417 struct ufs_vreg *vreg, bool on)
8418 {
8419 if (regulator_count_voltages(vreg->reg) <= 0)
8420 return 0;
8421
8422 return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
8423 }
8424
ufshcd_enable_vreg(struct device * dev,struct ufs_vreg * vreg)8425 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
8426 {
8427 int ret = 0;
8428
8429 if (!vreg || vreg->enabled)
8430 goto out;
8431
8432 ret = ufshcd_config_vreg(dev, vreg, true);
8433 if (!ret)
8434 ret = regulator_enable(vreg->reg);
8435
8436 if (!ret)
8437 vreg->enabled = true;
8438 else
8439 dev_err(dev, "%s: %s enable failed, err=%d\n",
8440 __func__, vreg->name, ret);
8441 out:
8442 return ret;
8443 }
8444
ufshcd_disable_vreg(struct device * dev,struct ufs_vreg * vreg)8445 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
8446 {
8447 int ret = 0;
8448
8449 if (!vreg || !vreg->enabled || vreg->always_on)
8450 goto out;
8451
8452 ret = regulator_disable(vreg->reg);
8453
8454 if (!ret) {
8455 /* ignore errors on applying disable config */
8456 ufshcd_config_vreg(dev, vreg, false);
8457 vreg->enabled = false;
8458 } else {
8459 dev_err(dev, "%s: %s disable failed, err=%d\n",
8460 __func__, vreg->name, ret);
8461 }
8462 out:
8463 return ret;
8464 }
8465
ufshcd_setup_vreg(struct ufs_hba * hba,bool on)8466 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
8467 {
8468 int ret = 0;
8469 struct device *dev = hba->dev;
8470 struct ufs_vreg_info *info = &hba->vreg_info;
8471
8472 ret = ufshcd_toggle_vreg(dev, info->vcc, on);
8473 if (ret)
8474 goto out;
8475
8476 ret = ufshcd_toggle_vreg(dev, info->vccq, on);
8477 if (ret)
8478 goto out;
8479
8480 ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
8481
8482 out:
8483 if (ret) {
8484 ufshcd_toggle_vreg(dev, info->vccq2, false);
8485 ufshcd_toggle_vreg(dev, info->vccq, false);
8486 ufshcd_toggle_vreg(dev, info->vcc, false);
8487 }
8488 return ret;
8489 }
8490
ufshcd_setup_hba_vreg(struct ufs_hba * hba,bool on)8491 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8492 {
8493 struct ufs_vreg_info *info = &hba->vreg_info;
8494
8495 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
8496 }
8497
ufshcd_get_vreg(struct device * dev,struct ufs_vreg * vreg)8498 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8499 {
8500 int ret = 0;
8501
8502 if (!vreg)
8503 goto out;
8504
8505 vreg->reg = devm_regulator_get(dev, vreg->name);
8506 if (IS_ERR(vreg->reg)) {
8507 ret = PTR_ERR(vreg->reg);
8508 dev_err(dev, "%s: %s get failed, err=%d\n",
8509 __func__, vreg->name, ret);
8510 }
8511 out:
8512 return ret;
8513 }
8514 EXPORT_SYMBOL_GPL(ufshcd_get_vreg);
8515
ufshcd_init_vreg(struct ufs_hba * hba)8516 static int ufshcd_init_vreg(struct ufs_hba *hba)
8517 {
8518 int ret = 0;
8519 struct device *dev = hba->dev;
8520 struct ufs_vreg_info *info = &hba->vreg_info;
8521
8522 ret = ufshcd_get_vreg(dev, info->vcc);
8523 if (ret)
8524 goto out;
8525
8526 ret = ufshcd_get_vreg(dev, info->vccq);
8527 if (!ret)
8528 ret = ufshcd_get_vreg(dev, info->vccq2);
8529 out:
8530 return ret;
8531 }
8532
ufshcd_init_hba_vreg(struct ufs_hba * hba)8533 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8534 {
8535 struct ufs_vreg_info *info = &hba->vreg_info;
8536
8537 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8538 }
8539
ufshcd_setup_clocks(struct ufs_hba * hba,bool on)8540 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8541 {
8542 int ret = 0;
8543 struct ufs_clk_info *clki;
8544 struct list_head *head = &hba->clk_list_head;
8545 unsigned long flags;
8546 ktime_t start = ktime_get();
8547 bool clk_state_changed = false;
8548
8549 if (list_empty(head))
8550 goto out;
8551
8552 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8553 if (ret)
8554 return ret;
8555
8556 list_for_each_entry(clki, head, list) {
8557 if (!IS_ERR_OR_NULL(clki->clk)) {
8558 /*
8559 * Don't disable clocks which are needed
8560 * to keep the link active.
8561 */
8562 if (ufshcd_is_link_active(hba) &&
8563 clki->keep_link_active)
8564 continue;
8565
8566 clk_state_changed = on ^ clki->enabled;
8567 if (on && !clki->enabled) {
8568 ret = clk_prepare_enable(clki->clk);
8569 if (ret) {
8570 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8571 __func__, clki->name, ret);
8572 goto out;
8573 }
8574 } else if (!on && clki->enabled) {
8575 clk_disable_unprepare(clki->clk);
8576 }
8577 clki->enabled = on;
8578 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8579 clki->name, on ? "en" : "dis");
8580 }
8581 }
8582
8583 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8584 if (ret)
8585 return ret;
8586
8587 out:
8588 if (ret) {
8589 list_for_each_entry(clki, head, list) {
8590 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8591 clk_disable_unprepare(clki->clk);
8592 }
8593 } else if (!ret && on) {
8594 spin_lock_irqsave(hba->host->host_lock, flags);
8595 hba->clk_gating.state = CLKS_ON;
8596 trace_ufshcd_clk_gating(dev_name(hba->dev),
8597 hba->clk_gating.state);
8598 spin_unlock_irqrestore(hba->host->host_lock, flags);
8599 }
8600
8601 if (clk_state_changed)
8602 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8603 (on ? "on" : "off"),
8604 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
8605 return ret;
8606 }
8607
ufshcd_parse_ref_clk_property(struct ufs_hba * hba)8608 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba)
8609 {
8610 u32 freq;
8611 int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq);
8612
8613 if (ret) {
8614 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret);
8615 return REF_CLK_FREQ_INVAL;
8616 }
8617
8618 return ufs_get_bref_clk_from_hz(freq);
8619 }
8620
ufshcd_init_clocks(struct ufs_hba * hba)8621 static int ufshcd_init_clocks(struct ufs_hba *hba)
8622 {
8623 int ret = 0;
8624 struct ufs_clk_info *clki;
8625 struct device *dev = hba->dev;
8626 struct list_head *head = &hba->clk_list_head;
8627
8628 if (list_empty(head))
8629 goto out;
8630
8631 list_for_each_entry(clki, head, list) {
8632 if (!clki->name)
8633 continue;
8634
8635 clki->clk = devm_clk_get(dev, clki->name);
8636 if (IS_ERR(clki->clk)) {
8637 ret = PTR_ERR(clki->clk);
8638 dev_err(dev, "%s: %s clk get failed, %d\n",
8639 __func__, clki->name, ret);
8640 goto out;
8641 }
8642
8643 /*
8644 * Parse device ref clk freq as per device tree "ref_clk".
8645 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8646 * in ufshcd_alloc_host().
8647 */
8648 if (!strcmp(clki->name, "ref_clk"))
8649 ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8650
8651 if (clki->max_freq) {
8652 ret = clk_set_rate(clki->clk, clki->max_freq);
8653 if (ret) {
8654 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8655 __func__, clki->name,
8656 clki->max_freq, ret);
8657 goto out;
8658 }
8659 clki->curr_freq = clki->max_freq;
8660 }
8661 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8662 clki->name, clk_get_rate(clki->clk));
8663 }
8664 out:
8665 return ret;
8666 }
8667
ufshcd_variant_hba_init(struct ufs_hba * hba)8668 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8669 {
8670 int err = 0;
8671
8672 if (!hba->vops)
8673 goto out;
8674
8675 err = ufshcd_vops_init(hba);
8676 if (err)
8677 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
8678 __func__, ufshcd_get_var_name(hba), err);
8679 out:
8680 return err;
8681 }
8682
ufshcd_variant_hba_exit(struct ufs_hba * hba)8683 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8684 {
8685 if (!hba->vops)
8686 return;
8687
8688 ufshcd_vops_exit(hba);
8689 }
8690
ufshcd_hba_init(struct ufs_hba * hba)8691 static int ufshcd_hba_init(struct ufs_hba *hba)
8692 {
8693 int err;
8694
8695 /*
8696 * Handle host controller power separately from the UFS device power
8697 * rails as it will help controlling the UFS host controller power
8698 * collapse easily which is different than UFS device power collapse.
8699 * Also, enable the host controller power before we go ahead with rest
8700 * of the initialization here.
8701 */
8702 err = ufshcd_init_hba_vreg(hba);
8703 if (err)
8704 goto out;
8705
8706 err = ufshcd_setup_hba_vreg(hba, true);
8707 if (err)
8708 goto out;
8709
8710 err = ufshcd_init_clocks(hba);
8711 if (err)
8712 goto out_disable_hba_vreg;
8713
8714 if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8715 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba);
8716
8717 err = ufshcd_setup_clocks(hba, true);
8718 if (err)
8719 goto out_disable_hba_vreg;
8720
8721 err = ufshcd_init_vreg(hba);
8722 if (err)
8723 goto out_disable_clks;
8724
8725 err = ufshcd_setup_vreg(hba, true);
8726 if (err)
8727 goto out_disable_clks;
8728
8729 err = ufshcd_variant_hba_init(hba);
8730 if (err)
8731 goto out_disable_vreg;
8732
8733 ufs_debugfs_hba_init(hba);
8734
8735 hba->is_powered = true;
8736 goto out;
8737
8738 out_disable_vreg:
8739 ufshcd_setup_vreg(hba, false);
8740 out_disable_clks:
8741 ufshcd_setup_clocks(hba, false);
8742 out_disable_hba_vreg:
8743 ufshcd_setup_hba_vreg(hba, false);
8744 out:
8745 return err;
8746 }
8747
ufshcd_hba_exit(struct ufs_hba * hba)8748 static void ufshcd_hba_exit(struct ufs_hba *hba)
8749 {
8750 if (hba->is_powered) {
8751 ufshcd_exit_clk_scaling(hba);
8752 ufshcd_exit_clk_gating(hba);
8753 if (hba->eh_wq)
8754 destroy_workqueue(hba->eh_wq);
8755 ufs_debugfs_hba_exit(hba);
8756 ufshcd_variant_hba_exit(hba);
8757 ufshcd_setup_vreg(hba, false);
8758 ufshcd_setup_clocks(hba, false);
8759 ufshcd_setup_hba_vreg(hba, false);
8760 hba->is_powered = false;
8761 ufs_put_device_desc(hba);
8762 }
8763 }
8764
8765 /**
8766 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8767 * power mode
8768 * @hba: per adapter instance
8769 * @pwr_mode: device power mode to set
8770 *
8771 * Returns 0 if requested power mode is set successfully
8772 * Returns < 0 if failed to set the requested power mode
8773 */
ufshcd_set_dev_pwr_mode(struct ufs_hba * hba,enum ufs_dev_pwr_mode pwr_mode)8774 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8775 enum ufs_dev_pwr_mode pwr_mode)
8776 {
8777 unsigned char cmd[6] = { START_STOP };
8778 struct scsi_sense_hdr sshdr;
8779 struct scsi_device *sdp;
8780 unsigned long flags;
8781 int ret, retries;
8782
8783 spin_lock_irqsave(hba->host->host_lock, flags);
8784 sdp = hba->ufs_device_wlun;
8785 if (sdp) {
8786 ret = scsi_device_get(sdp);
8787 if (!ret && !scsi_device_online(sdp)) {
8788 ret = -ENODEV;
8789 scsi_device_put(sdp);
8790 }
8791 } else {
8792 ret = -ENODEV;
8793 }
8794 spin_unlock_irqrestore(hba->host->host_lock, flags);
8795
8796 if (ret)
8797 return ret;
8798
8799 /*
8800 * If scsi commands fail, the scsi mid-layer schedules scsi error-
8801 * handling, which would wait for host to be resumed. Since we know
8802 * we are functional while we are here, skip host resume in error
8803 * handling context.
8804 */
8805 hba->host->eh_noresume = 1;
8806
8807 cmd[4] = pwr_mode << 4;
8808
8809 /*
8810 * Current function would be generally called from the power management
8811 * callbacks hence set the RQF_PM flag so that it doesn't resume the
8812 * already suspended childs.
8813 */
8814 for (retries = 3; retries > 0; --retries) {
8815 ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8816 HZ, 0, 0, RQF_PM, NULL);
8817 if (!scsi_status_is_check_condition(ret) ||
8818 !scsi_sense_valid(&sshdr) ||
8819 sshdr.sense_key != UNIT_ATTENTION)
8820 break;
8821 }
8822 if (ret) {
8823 sdev_printk(KERN_WARNING, sdp,
8824 "START_STOP failed for power mode: %d, result %x\n",
8825 pwr_mode, ret);
8826 if (ret > 0) {
8827 if (scsi_sense_valid(&sshdr))
8828 scsi_print_sense_hdr(sdp, NULL, &sshdr);
8829 ret = -EIO;
8830 }
8831 }
8832
8833 if (!ret)
8834 hba->curr_dev_pwr_mode = pwr_mode;
8835
8836 scsi_device_put(sdp);
8837 hba->host->eh_noresume = 0;
8838 return ret;
8839 }
8840
ufshcd_link_state_transition(struct ufs_hba * hba,enum uic_link_state req_link_state,int check_for_bkops)8841 static int ufshcd_link_state_transition(struct ufs_hba *hba,
8842 enum uic_link_state req_link_state,
8843 int check_for_bkops)
8844 {
8845 int ret = 0;
8846
8847 if (req_link_state == hba->uic_link_state)
8848 return 0;
8849
8850 if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8851 ret = ufshcd_uic_hibern8_enter(hba);
8852 if (!ret) {
8853 ufshcd_set_link_hibern8(hba);
8854 } else {
8855 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8856 __func__, ret);
8857 goto out;
8858 }
8859 }
8860 /*
8861 * If autobkops is enabled, link can't be turned off because
8862 * turning off the link would also turn off the device, except in the
8863 * case of DeepSleep where the device is expected to remain powered.
8864 */
8865 else if ((req_link_state == UIC_LINK_OFF_STATE) &&
8866 (!check_for_bkops || !hba->auto_bkops_enabled)) {
8867 /*
8868 * Let's make sure that link is in low power mode, we are doing
8869 * this currently by putting the link in Hibern8. Otherway to
8870 * put the link in low power mode is to send the DME end point
8871 * to device and then send the DME reset command to local
8872 * unipro. But putting the link in hibern8 is much faster.
8873 *
8874 * Note also that putting the link in Hibern8 is a requirement
8875 * for entering DeepSleep.
8876 */
8877 ret = ufshcd_uic_hibern8_enter(hba);
8878 if (ret) {
8879 dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8880 __func__, ret);
8881 goto out;
8882 }
8883 /*
8884 * Change controller state to "reset state" which
8885 * should also put the link in off/reset state
8886 */
8887 ufshcd_hba_stop(hba);
8888 /*
8889 * TODO: Check if we need any delay to make sure that
8890 * controller is reset
8891 */
8892 ufshcd_set_link_off(hba);
8893 }
8894
8895 out:
8896 return ret;
8897 }
8898
ufshcd_vreg_set_lpm(struct ufs_hba * hba)8899 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8900 {
8901 bool vcc_off = false;
8902
8903 /*
8904 * It seems some UFS devices may keep drawing more than sleep current
8905 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8906 * To avoid this situation, add 2ms delay before putting these UFS
8907 * rails in LPM mode.
8908 */
8909 if (!ufshcd_is_link_active(hba) &&
8910 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8911 usleep_range(2000, 2100);
8912
8913 /*
8914 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8915 * power.
8916 *
8917 * If UFS device and link is in OFF state, all power supplies (VCC,
8918 * VCCQ, VCCQ2) can be turned off if power on write protect is not
8919 * required. If UFS link is inactive (Hibern8 or OFF state) and device
8920 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8921 *
8922 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8923 * in low power state which would save some power.
8924 *
8925 * If Write Booster is enabled and the device needs to flush the WB
8926 * buffer OR if bkops status is urgent for WB, keep Vcc on.
8927 */
8928 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8929 !hba->dev_info.is_lu_power_on_wp) {
8930 ufshcd_setup_vreg(hba, false);
8931 vcc_off = true;
8932 } else if (!ufshcd_is_ufs_dev_active(hba)) {
8933 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8934 vcc_off = true;
8935 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
8936 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8937 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8938 }
8939 }
8940
8941 /*
8942 * Some UFS devices require delay after VCC power rail is turned-off.
8943 */
8944 if (vcc_off && hba->vreg_info.vcc &&
8945 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8946 usleep_range(5000, 5100);
8947 }
8948
8949 #ifdef CONFIG_PM
ufshcd_vreg_set_hpm(struct ufs_hba * hba)8950 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8951 {
8952 int ret = 0;
8953
8954 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8955 !hba->dev_info.is_lu_power_on_wp) {
8956 ret = ufshcd_setup_vreg(hba, true);
8957 } else if (!ufshcd_is_ufs_dev_active(hba)) {
8958 if (!ufshcd_is_link_active(hba)) {
8959 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8960 if (ret)
8961 goto vcc_disable;
8962 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8963 if (ret)
8964 goto vccq_lpm;
8965 }
8966 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
8967 }
8968 goto out;
8969
8970 vccq_lpm:
8971 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8972 vcc_disable:
8973 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8974 out:
8975 return ret;
8976 }
8977 #endif /* CONFIG_PM */
8978
ufshcd_hba_vreg_set_lpm(struct ufs_hba * hba)8979 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8980 {
8981 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8982 ufshcd_setup_hba_vreg(hba, false);
8983 }
8984
ufshcd_hba_vreg_set_hpm(struct ufs_hba * hba)8985 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8986 {
8987 if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8988 ufshcd_setup_hba_vreg(hba, true);
8989 }
8990
__ufshcd_wl_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op)8991 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8992 {
8993 int ret = 0;
8994 int check_for_bkops;
8995 enum ufs_pm_level pm_lvl;
8996 enum ufs_dev_pwr_mode req_dev_pwr_mode;
8997 enum uic_link_state req_link_state;
8998
8999 hba->pm_op_in_progress = true;
9000 if (pm_op != UFS_SHUTDOWN_PM) {
9001 pm_lvl = pm_op == UFS_RUNTIME_PM ?
9002 hba->rpm_lvl : hba->spm_lvl;
9003 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
9004 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
9005 } else {
9006 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
9007 req_link_state = UIC_LINK_OFF_STATE;
9008 }
9009
9010 ufshpb_suspend(hba);
9011
9012 /*
9013 * If we can't transition into any of the low power modes
9014 * just gate the clocks.
9015 */
9016 ufshcd_hold(hba, false);
9017 hba->clk_gating.is_suspended = true;
9018
9019 if (ufshcd_is_clkscaling_supported(hba))
9020 ufshcd_clk_scaling_suspend(hba, true);
9021
9022 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
9023 req_link_state == UIC_LINK_ACTIVE_STATE) {
9024 goto vops_suspend;
9025 }
9026
9027 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
9028 (req_link_state == hba->uic_link_state))
9029 goto enable_scaling;
9030
9031 /* UFS device & link must be active before we enter in this function */
9032 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
9033 ret = -EINVAL;
9034 goto enable_scaling;
9035 }
9036
9037 if (pm_op == UFS_RUNTIME_PM) {
9038 if (ufshcd_can_autobkops_during_suspend(hba)) {
9039 /*
9040 * The device is idle with no requests in the queue,
9041 * allow background operations if bkops status shows
9042 * that performance might be impacted.
9043 */
9044 ret = ufshcd_urgent_bkops(hba);
9045 if (ret)
9046 goto enable_scaling;
9047 } else {
9048 /* make sure that auto bkops is disabled */
9049 ufshcd_disable_auto_bkops(hba);
9050 }
9051 /*
9052 * If device needs to do BKOP or WB buffer flush during
9053 * Hibern8, keep device power mode as "active power mode"
9054 * and VCC supply.
9055 */
9056 hba->dev_info.b_rpm_dev_flush_capable =
9057 hba->auto_bkops_enabled ||
9058 (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
9059 ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
9060 ufshcd_is_auto_hibern8_enabled(hba))) &&
9061 ufshcd_wb_need_flush(hba));
9062 }
9063
9064 flush_work(&hba->eeh_work);
9065
9066 ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9067 if (ret)
9068 goto enable_scaling;
9069
9070 if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9071 if (pm_op != UFS_RUNTIME_PM)
9072 /* ensure that bkops is disabled */
9073 ufshcd_disable_auto_bkops(hba);
9074
9075 if (!hba->dev_info.b_rpm_dev_flush_capable) {
9076 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
9077 if (ret && pm_op != UFS_SHUTDOWN_PM) {
9078 /*
9079 * If return err in suspend flow, IO will hang.
9080 * Trigger error handler and break suspend for
9081 * error recovery.
9082 */
9083 ufshcd_force_error_recovery(hba);
9084 ret = -EBUSY;
9085 }
9086 if (ret)
9087 goto enable_scaling;
9088 }
9089 }
9090
9091 /*
9092 * In the case of DeepSleep, the device is expected to remain powered
9093 * with the link off, so do not check for bkops.
9094 */
9095 check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
9096 ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
9097 if (ret && pm_op != UFS_SHUTDOWN_PM) {
9098 /*
9099 * If return err in suspend flow, IO will hang.
9100 * Trigger error handler and break suspend for
9101 * error recovery.
9102 */
9103 ufshcd_force_error_recovery(hba);
9104 ret = -EBUSY;
9105 }
9106 if (ret)
9107 goto set_dev_active;
9108
9109 vops_suspend:
9110 /*
9111 * Call vendor specific suspend callback. As these callbacks may access
9112 * vendor specific host controller register space call them before the
9113 * host clocks are ON.
9114 */
9115 ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9116 if (ret)
9117 goto set_link_active;
9118 goto out;
9119
9120 set_link_active:
9121 /*
9122 * Device hardware reset is required to exit DeepSleep. Also, for
9123 * DeepSleep, the link is off so host reset and restore will be done
9124 * further below.
9125 */
9126 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9127 ufshcd_device_reset(hba);
9128 WARN_ON(!ufshcd_is_link_off(hba));
9129 }
9130 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9131 ufshcd_set_link_active(hba);
9132 else if (ufshcd_is_link_off(hba))
9133 ufshcd_host_reset_and_restore(hba);
9134 set_dev_active:
9135 /* Can also get here needing to exit DeepSleep */
9136 if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9137 ufshcd_device_reset(hba);
9138 ufshcd_host_reset_and_restore(hba);
9139 }
9140 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9141 ufshcd_disable_auto_bkops(hba);
9142 enable_scaling:
9143 if (ufshcd_is_clkscaling_supported(hba))
9144 ufshcd_clk_scaling_suspend(hba, false);
9145
9146 hba->dev_info.b_rpm_dev_flush_capable = false;
9147 out:
9148 if (hba->dev_info.b_rpm_dev_flush_capable) {
9149 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9150 msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9151 }
9152
9153 if (ret) {
9154 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9155 hba->clk_gating.is_suspended = false;
9156 ufshcd_release(hba);
9157 ufshpb_resume(hba);
9158 }
9159 hba->pm_op_in_progress = false;
9160 return ret;
9161 }
9162
9163 #ifdef CONFIG_PM
__ufshcd_wl_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)9164 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9165 {
9166 int ret;
9167 enum uic_link_state old_link_state = hba->uic_link_state;
9168
9169 hba->pm_op_in_progress = true;
9170
9171 /*
9172 * Call vendor specific resume callback. As these callbacks may access
9173 * vendor specific host controller register space call them when the
9174 * host clocks are ON.
9175 */
9176 ret = ufshcd_vops_resume(hba, pm_op);
9177 if (ret)
9178 goto out;
9179
9180 /* For DeepSleep, the only supported option is to have the link off */
9181 WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9182
9183 if (ufshcd_is_link_hibern8(hba)) {
9184 ret = ufshcd_uic_hibern8_exit(hba);
9185 if (!ret) {
9186 ufshcd_set_link_active(hba);
9187 } else {
9188 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9189 __func__, ret);
9190 goto vendor_suspend;
9191 }
9192 } else if (ufshcd_is_link_off(hba)) {
9193 /*
9194 * A full initialization of the host and the device is
9195 * required since the link was put to off during suspend.
9196 * Note, in the case of DeepSleep, the device will exit
9197 * DeepSleep due to device reset.
9198 */
9199 ret = ufshcd_reset_and_restore(hba);
9200 /*
9201 * ufshcd_reset_and_restore() should have already
9202 * set the link state as active
9203 */
9204 if (ret || !ufshcd_is_link_active(hba))
9205 goto vendor_suspend;
9206 }
9207
9208 if (!ufshcd_is_ufs_dev_active(hba)) {
9209 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9210 if (ret)
9211 goto set_old_link_state;
9212 }
9213
9214 if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9215 ufshcd_enable_auto_bkops(hba);
9216 else
9217 /*
9218 * If BKOPs operations are urgently needed at this moment then
9219 * keep auto-bkops enabled or else disable it.
9220 */
9221 ufshcd_urgent_bkops(hba);
9222
9223 if (hba->ee_usr_mask)
9224 ufshcd_write_ee_control(hba);
9225
9226 if (ufshcd_is_clkscaling_supported(hba))
9227 ufshcd_clk_scaling_suspend(hba, false);
9228
9229 if (hba->dev_info.b_rpm_dev_flush_capable) {
9230 hba->dev_info.b_rpm_dev_flush_capable = false;
9231 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9232 }
9233
9234 /* Enable Auto-Hibernate if configured */
9235 ufshcd_auto_hibern8_enable(hba);
9236
9237 ufshpb_resume(hba);
9238 goto out;
9239
9240 set_old_link_state:
9241 ufshcd_link_state_transition(hba, old_link_state, 0);
9242 vendor_suspend:
9243 ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9244 ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9245 out:
9246 if (ret)
9247 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9248 hba->clk_gating.is_suspended = false;
9249 ufshcd_release(hba);
9250 hba->pm_op_in_progress = false;
9251 return ret;
9252 }
9253
ufshcd_wl_runtime_suspend(struct device * dev)9254 static int ufshcd_wl_runtime_suspend(struct device *dev)
9255 {
9256 struct scsi_device *sdev = to_scsi_device(dev);
9257 struct ufs_hba *hba;
9258 int ret;
9259 ktime_t start = ktime_get();
9260
9261 hba = shost_priv(sdev->host);
9262
9263 ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9264 if (ret)
9265 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9266
9267 trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9268 ktime_to_us(ktime_sub(ktime_get(), start)),
9269 hba->curr_dev_pwr_mode, hba->uic_link_state);
9270
9271 return ret;
9272 }
9273
ufshcd_wl_runtime_resume(struct device * dev)9274 static int ufshcd_wl_runtime_resume(struct device *dev)
9275 {
9276 struct scsi_device *sdev = to_scsi_device(dev);
9277 struct ufs_hba *hba;
9278 int ret = 0;
9279 ktime_t start = ktime_get();
9280
9281 hba = shost_priv(sdev->host);
9282
9283 ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9284 if (ret)
9285 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9286
9287 trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9288 ktime_to_us(ktime_sub(ktime_get(), start)),
9289 hba->curr_dev_pwr_mode, hba->uic_link_state);
9290
9291 return ret;
9292 }
9293 #endif
9294
9295 #ifdef CONFIG_PM_SLEEP
ufshcd_wl_suspend(struct device * dev)9296 static int ufshcd_wl_suspend(struct device *dev)
9297 {
9298 struct scsi_device *sdev = to_scsi_device(dev);
9299 struct ufs_hba *hba;
9300 int ret = 0;
9301 ktime_t start = ktime_get();
9302
9303 hba = shost_priv(sdev->host);
9304 down(&hba->host_sem);
9305
9306 if (pm_runtime_suspended(dev))
9307 goto out;
9308
9309 ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9310 if (ret) {
9311 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9312 up(&hba->host_sem);
9313 }
9314
9315 out:
9316 if (!ret)
9317 hba->is_sys_suspended = true;
9318 trace_ufshcd_wl_suspend(dev_name(dev), ret,
9319 ktime_to_us(ktime_sub(ktime_get(), start)),
9320 hba->curr_dev_pwr_mode, hba->uic_link_state);
9321
9322 return ret;
9323 }
9324
ufshcd_wl_resume(struct device * dev)9325 static int ufshcd_wl_resume(struct device *dev)
9326 {
9327 struct scsi_device *sdev = to_scsi_device(dev);
9328 struct ufs_hba *hba;
9329 int ret = 0;
9330 ktime_t start = ktime_get();
9331
9332 hba = shost_priv(sdev->host);
9333
9334 if (pm_runtime_suspended(dev))
9335 goto out;
9336
9337 ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9338 if (ret)
9339 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9340 out:
9341 trace_ufshcd_wl_resume(dev_name(dev), ret,
9342 ktime_to_us(ktime_sub(ktime_get(), start)),
9343 hba->curr_dev_pwr_mode, hba->uic_link_state);
9344 if (!ret)
9345 hba->is_sys_suspended = false;
9346 up(&hba->host_sem);
9347 return ret;
9348 }
9349 #endif
9350
ufshcd_wl_shutdown(struct device * dev)9351 static void ufshcd_wl_shutdown(struct device *dev)
9352 {
9353 struct scsi_device *sdev = to_scsi_device(dev);
9354 struct ufs_hba *hba;
9355
9356 hba = shost_priv(sdev->host);
9357
9358 down(&hba->host_sem);
9359 hba->shutting_down = true;
9360 up(&hba->host_sem);
9361
9362 /* Turn on everything while shutting down */
9363 ufshcd_rpm_get_sync(hba);
9364 scsi_device_quiesce(sdev);
9365 shost_for_each_device(sdev, hba->host) {
9366 if (sdev == hba->ufs_device_wlun)
9367 continue;
9368 scsi_device_quiesce(sdev);
9369 }
9370 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9371 }
9372
9373 /**
9374 * ufshcd_suspend - helper function for suspend operations
9375 * @hba: per adapter instance
9376 *
9377 * This function will put disable irqs, turn off clocks
9378 * and set vreg and hba-vreg in lpm mode.
9379 */
ufshcd_suspend(struct ufs_hba * hba)9380 static int ufshcd_suspend(struct ufs_hba *hba)
9381 {
9382 int ret;
9383
9384 if (!hba->is_powered)
9385 return 0;
9386 /*
9387 * Disable the host irq as host controller as there won't be any
9388 * host controller transaction expected till resume.
9389 */
9390 ufshcd_disable_irq(hba);
9391 ret = ufshcd_setup_clocks(hba, false);
9392 if (ret) {
9393 ufshcd_enable_irq(hba);
9394 return ret;
9395 }
9396 if (ufshcd_is_clkgating_allowed(hba)) {
9397 hba->clk_gating.state = CLKS_OFF;
9398 trace_ufshcd_clk_gating(dev_name(hba->dev),
9399 hba->clk_gating.state);
9400 }
9401
9402 ufshcd_vreg_set_lpm(hba);
9403 /* Put the host controller in low power mode if possible */
9404 ufshcd_hba_vreg_set_lpm(hba);
9405 return ret;
9406 }
9407
9408 #ifdef CONFIG_PM
9409 /**
9410 * ufshcd_resume - helper function for resume operations
9411 * @hba: per adapter instance
9412 *
9413 * This function basically turns on the regulators, clocks and
9414 * irqs of the hba.
9415 *
9416 * Returns 0 for success and non-zero for failure
9417 */
ufshcd_resume(struct ufs_hba * hba)9418 static int ufshcd_resume(struct ufs_hba *hba)
9419 {
9420 int ret;
9421
9422 if (!hba->is_powered)
9423 return 0;
9424
9425 ufshcd_hba_vreg_set_hpm(hba);
9426 ret = ufshcd_vreg_set_hpm(hba);
9427 if (ret)
9428 goto out;
9429
9430 /* Make sure clocks are enabled before accessing controller */
9431 ret = ufshcd_setup_clocks(hba, true);
9432 if (ret)
9433 goto disable_vreg;
9434
9435 /* enable the host irq as host controller would be active soon */
9436 ufshcd_enable_irq(hba);
9437 goto out;
9438
9439 disable_vreg:
9440 ufshcd_vreg_set_lpm(hba);
9441 out:
9442 if (ret)
9443 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
9444 return ret;
9445 }
9446 #endif /* CONFIG_PM */
9447
9448 #ifdef CONFIG_PM_SLEEP
9449 /**
9450 * ufshcd_system_suspend - system suspend callback
9451 * @dev: Device associated with the UFS controller.
9452 *
9453 * Executed before putting the system into a sleep state in which the contents
9454 * of main memory are preserved.
9455 *
9456 * Returns 0 for success and non-zero for failure
9457 */
ufshcd_system_suspend(struct device * dev)9458 int ufshcd_system_suspend(struct device *dev)
9459 {
9460 struct ufs_hba *hba = dev_get_drvdata(dev);
9461 int ret = 0;
9462 ktime_t start = ktime_get();
9463
9464 if (pm_runtime_suspended(hba->dev))
9465 goto out;
9466
9467 ret = ufshcd_suspend(hba);
9468 out:
9469 trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
9470 ktime_to_us(ktime_sub(ktime_get(), start)),
9471 hba->curr_dev_pwr_mode, hba->uic_link_state);
9472 return ret;
9473 }
9474 EXPORT_SYMBOL(ufshcd_system_suspend);
9475
9476 /**
9477 * ufshcd_system_resume - system resume callback
9478 * @dev: Device associated with the UFS controller.
9479 *
9480 * Executed after waking the system up from a sleep state in which the contents
9481 * of main memory were preserved.
9482 *
9483 * Returns 0 for success and non-zero for failure
9484 */
ufshcd_system_resume(struct device * dev)9485 int ufshcd_system_resume(struct device *dev)
9486 {
9487 struct ufs_hba *hba = dev_get_drvdata(dev);
9488 ktime_t start = ktime_get();
9489 int ret = 0;
9490
9491 if (pm_runtime_suspended(hba->dev))
9492 goto out;
9493
9494 ret = ufshcd_resume(hba);
9495
9496 out:
9497 trace_ufshcd_system_resume(dev_name(hba->dev), ret,
9498 ktime_to_us(ktime_sub(ktime_get(), start)),
9499 hba->curr_dev_pwr_mode, hba->uic_link_state);
9500
9501 return ret;
9502 }
9503 EXPORT_SYMBOL(ufshcd_system_resume);
9504 #endif /* CONFIG_PM_SLEEP */
9505
9506 #ifdef CONFIG_PM
9507 /**
9508 * ufshcd_runtime_suspend - runtime suspend callback
9509 * @dev: Device associated with the UFS controller.
9510 *
9511 * Check the description of ufshcd_suspend() function for more details.
9512 *
9513 * Returns 0 for success and non-zero for failure
9514 */
ufshcd_runtime_suspend(struct device * dev)9515 int ufshcd_runtime_suspend(struct device *dev)
9516 {
9517 struct ufs_hba *hba = dev_get_drvdata(dev);
9518 int ret;
9519 ktime_t start = ktime_get();
9520
9521 ret = ufshcd_suspend(hba);
9522
9523 trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
9524 ktime_to_us(ktime_sub(ktime_get(), start)),
9525 hba->curr_dev_pwr_mode, hba->uic_link_state);
9526 return ret;
9527 }
9528 EXPORT_SYMBOL(ufshcd_runtime_suspend);
9529
9530 /**
9531 * ufshcd_runtime_resume - runtime resume routine
9532 * @dev: Device associated with the UFS controller.
9533 *
9534 * This function basically brings controller
9535 * to active state. Following operations are done in this function:
9536 *
9537 * 1. Turn on all the controller related clocks
9538 * 2. Turn ON VCC rail
9539 */
ufshcd_runtime_resume(struct device * dev)9540 int ufshcd_runtime_resume(struct device *dev)
9541 {
9542 struct ufs_hba *hba = dev_get_drvdata(dev);
9543 int ret;
9544 ktime_t start = ktime_get();
9545
9546 ret = ufshcd_resume(hba);
9547
9548 trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
9549 ktime_to_us(ktime_sub(ktime_get(), start)),
9550 hba->curr_dev_pwr_mode, hba->uic_link_state);
9551 return ret;
9552 }
9553 EXPORT_SYMBOL(ufshcd_runtime_resume);
9554 #endif /* CONFIG_PM */
9555
9556 /**
9557 * ufshcd_shutdown - shutdown routine
9558 * @hba: per adapter instance
9559 *
9560 * This function would turn off both UFS device and UFS hba
9561 * regulators. It would also disable clocks.
9562 *
9563 * Returns 0 always to allow force shutdown even in case of errors.
9564 */
ufshcd_shutdown(struct ufs_hba * hba)9565 int ufshcd_shutdown(struct ufs_hba *hba)
9566 {
9567 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9568 ufshcd_suspend(hba);
9569
9570 hba->is_powered = false;
9571 /* allow force shutdown even in case of errors */
9572 return 0;
9573 }
9574 EXPORT_SYMBOL(ufshcd_shutdown);
9575
9576 /**
9577 * ufshcd_remove - de-allocate SCSI host and host memory space
9578 * data structure memory
9579 * @hba: per adapter instance
9580 */
ufshcd_remove(struct ufs_hba * hba)9581 void ufshcd_remove(struct ufs_hba *hba)
9582 {
9583 if (hba->ufs_device_wlun)
9584 ufshcd_rpm_get_sync(hba);
9585 ufs_hwmon_remove(hba);
9586 ufs_bsg_remove(hba);
9587 ufshpb_remove(hba);
9588 ufs_sysfs_remove_nodes(hba->dev);
9589 blk_mq_destroy_queue(hba->tmf_queue);
9590 blk_mq_free_tag_set(&hba->tmf_tag_set);
9591 scsi_remove_host(hba->host);
9592 /* disable interrupts */
9593 ufshcd_disable_intr(hba, hba->intr_mask);
9594 ufshcd_hba_stop(hba);
9595 ufshcd_hba_exit(hba);
9596 }
9597 EXPORT_SYMBOL_GPL(ufshcd_remove);
9598
9599 /**
9600 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9601 * @hba: pointer to Host Bus Adapter (HBA)
9602 */
ufshcd_dealloc_host(struct ufs_hba * hba)9603 void ufshcd_dealloc_host(struct ufs_hba *hba)
9604 {
9605 scsi_host_put(hba->host);
9606 }
9607 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9608
9609 /**
9610 * ufshcd_set_dma_mask - Set dma mask based on the controller
9611 * addressing capability
9612 * @hba: per adapter instance
9613 *
9614 * Returns 0 for success, non-zero for failure
9615 */
ufshcd_set_dma_mask(struct ufs_hba * hba)9616 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9617 {
9618 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9619 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9620 return 0;
9621 }
9622 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9623 }
9624
9625 /**
9626 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
9627 * @dev: pointer to device handle
9628 * @hba_handle: driver private handle
9629 * Returns 0 on success, non-zero value on failure
9630 */
ufshcd_alloc_host(struct device * dev,struct ufs_hba ** hba_handle)9631 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
9632 {
9633 struct Scsi_Host *host;
9634 struct ufs_hba *hba;
9635 int err = 0;
9636
9637 if (!dev) {
9638 dev_err(dev,
9639 "Invalid memory reference for dev is NULL\n");
9640 err = -ENODEV;
9641 goto out_error;
9642 }
9643
9644 host = scsi_host_alloc(&ufshcd_driver_template,
9645 sizeof(struct ufs_hba));
9646 if (!host) {
9647 dev_err(dev, "scsi_host_alloc failed\n");
9648 err = -ENOMEM;
9649 goto out_error;
9650 }
9651 host->nr_maps = HCTX_TYPE_POLL + 1;
9652 hba = shost_priv(host);
9653 hba->host = host;
9654 hba->dev = dev;
9655 hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
9656 hba->nop_out_timeout = NOP_OUT_TIMEOUT;
9657 INIT_LIST_HEAD(&hba->clk_list_head);
9658 spin_lock_init(&hba->outstanding_lock);
9659
9660 *hba_handle = hba;
9661
9662 out_error:
9663 return err;
9664 }
9665 EXPORT_SYMBOL(ufshcd_alloc_host);
9666
9667 /* This function exists because blk_mq_alloc_tag_set() requires this. */
ufshcd_queue_tmf(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * qd)9668 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9669 const struct blk_mq_queue_data *qd)
9670 {
9671 WARN_ON_ONCE(true);
9672 return BLK_STS_NOTSUPP;
9673 }
9674
9675 static const struct blk_mq_ops ufshcd_tmf_ops = {
9676 .queue_rq = ufshcd_queue_tmf,
9677 };
9678
9679 /**
9680 * ufshcd_init - Driver initialization routine
9681 * @hba: per-adapter instance
9682 * @mmio_base: base register address
9683 * @irq: Interrupt line of device
9684 * Returns 0 on success, non-zero value on failure
9685 */
ufshcd_init(struct ufs_hba * hba,void __iomem * mmio_base,unsigned int irq)9686 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9687 {
9688 int err;
9689 struct Scsi_Host *host = hba->host;
9690 struct device *dev = hba->dev;
9691 char eh_wq_name[sizeof("ufs_eh_wq_00")];
9692
9693 /*
9694 * dev_set_drvdata() must be called before any callbacks are registered
9695 * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
9696 * sysfs).
9697 */
9698 dev_set_drvdata(dev, hba);
9699
9700 if (!mmio_base) {
9701 dev_err(hba->dev,
9702 "Invalid memory reference for mmio_base is NULL\n");
9703 err = -ENODEV;
9704 goto out_error;
9705 }
9706
9707 hba->mmio_base = mmio_base;
9708 hba->irq = irq;
9709 hba->vps = &ufs_hba_vps;
9710
9711 err = ufshcd_hba_init(hba);
9712 if (err)
9713 goto out_error;
9714
9715 /* Read capabilities registers */
9716 err = ufshcd_hba_capabilities(hba);
9717 if (err)
9718 goto out_disable;
9719
9720 /* Get UFS version supported by the controller */
9721 hba->ufs_version = ufshcd_get_ufs_version(hba);
9722
9723 /* Get Interrupt bit mask per version */
9724 hba->intr_mask = ufshcd_get_intr_mask(hba);
9725
9726 err = ufshcd_set_dma_mask(hba);
9727 if (err) {
9728 dev_err(hba->dev, "set dma mask failed\n");
9729 goto out_disable;
9730 }
9731
9732 /* Allocate memory for host memory space */
9733 err = ufshcd_memory_alloc(hba);
9734 if (err) {
9735 dev_err(hba->dev, "Memory allocation failed\n");
9736 goto out_disable;
9737 }
9738
9739 /* Configure LRB */
9740 ufshcd_host_memory_configure(hba);
9741
9742 host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
9743 host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
9744 host->max_id = UFSHCD_MAX_ID;
9745 host->max_lun = UFS_MAX_LUNS;
9746 host->max_channel = UFSHCD_MAX_CHANNEL;
9747 host->unique_id = host->host_no;
9748 host->max_cmd_len = UFS_CDB_SIZE;
9749
9750 hba->max_pwr_info.is_valid = false;
9751
9752 /* Initialize work queues */
9753 snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9754 hba->host->host_no);
9755 hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9756 if (!hba->eh_wq) {
9757 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9758 __func__);
9759 err = -ENOMEM;
9760 goto out_disable;
9761 }
9762 INIT_WORK(&hba->eh_work, ufshcd_err_handler);
9763 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
9764
9765 sema_init(&hba->host_sem, 1);
9766
9767 /* Initialize UIC command mutex */
9768 mutex_init(&hba->uic_cmd_mutex);
9769
9770 /* Initialize mutex for device management commands */
9771 mutex_init(&hba->dev_cmd.lock);
9772
9773 /* Initialize mutex for exception event control */
9774 mutex_init(&hba->ee_ctrl_mutex);
9775
9776 mutex_init(&hba->wb_mutex);
9777 init_rwsem(&hba->clk_scaling_lock);
9778
9779 ufshcd_init_clk_gating(hba);
9780
9781 ufshcd_init_clk_scaling(hba);
9782
9783 /*
9784 * In order to avoid any spurious interrupt immediately after
9785 * registering UFS controller interrupt handler, clear any pending UFS
9786 * interrupt status and disable all the UFS interrupts.
9787 */
9788 ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9789 REG_INTERRUPT_STATUS);
9790 ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9791 /*
9792 * Make sure that UFS interrupts are disabled and any pending interrupt
9793 * status is cleared before registering UFS interrupt handler.
9794 */
9795 mb();
9796
9797 /* IRQ registration */
9798 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
9799 if (err) {
9800 dev_err(hba->dev, "request irq failed\n");
9801 goto out_disable;
9802 } else {
9803 hba->is_irq_enabled = true;
9804 }
9805
9806 err = scsi_add_host(host, hba->dev);
9807 if (err) {
9808 dev_err(hba->dev, "scsi_add_host failed\n");
9809 goto out_disable;
9810 }
9811
9812 hba->tmf_tag_set = (struct blk_mq_tag_set) {
9813 .nr_hw_queues = 1,
9814 .queue_depth = hba->nutmrs,
9815 .ops = &ufshcd_tmf_ops,
9816 .flags = BLK_MQ_F_NO_SCHED,
9817 };
9818 err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9819 if (err < 0)
9820 goto out_remove_scsi_host;
9821 hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9822 if (IS_ERR(hba->tmf_queue)) {
9823 err = PTR_ERR(hba->tmf_queue);
9824 goto free_tmf_tag_set;
9825 }
9826 hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
9827 sizeof(*hba->tmf_rqs), GFP_KERNEL);
9828 if (!hba->tmf_rqs) {
9829 err = -ENOMEM;
9830 goto free_tmf_queue;
9831 }
9832
9833 /* Reset the attached device */
9834 ufshcd_device_reset(hba);
9835
9836 ufshcd_init_crypto(hba);
9837
9838 /* Host controller enable */
9839 err = ufshcd_hba_enable(hba);
9840 if (err) {
9841 dev_err(hba->dev, "Host controller enable failed\n");
9842 ufshcd_print_evt_hist(hba);
9843 ufshcd_print_host_state(hba);
9844 goto free_tmf_queue;
9845 }
9846
9847 /*
9848 * Set the default power management level for runtime and system PM.
9849 * Default power saving mode is to keep UFS link in Hibern8 state
9850 * and UFS device in sleep state.
9851 */
9852 hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9853 UFS_SLEEP_PWR_MODE,
9854 UIC_LINK_HIBERN8_STATE);
9855 hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9856 UFS_SLEEP_PWR_MODE,
9857 UIC_LINK_HIBERN8_STATE);
9858
9859 INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9860 ufshcd_rpm_dev_flush_recheck_work);
9861
9862 /* Set the default auto-hiberate idle timer value to 150 ms */
9863 if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
9864 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9865 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9866 }
9867
9868 /* Hold auto suspend until async scan completes */
9869 pm_runtime_get_sync(dev);
9870 atomic_set(&hba->scsi_block_reqs_cnt, 0);
9871 /*
9872 * We are assuming that device wasn't put in sleep/power-down
9873 * state exclusively during the boot stage before kernel.
9874 * This assumption helps avoid doing link startup twice during
9875 * ufshcd_probe_hba().
9876 */
9877 ufshcd_set_ufs_dev_active(hba);
9878
9879 async_schedule(ufshcd_async_scan, hba);
9880 ufs_sysfs_add_nodes(hba->dev);
9881
9882 device_enable_async_suspend(dev);
9883 return 0;
9884
9885 free_tmf_queue:
9886 blk_mq_destroy_queue(hba->tmf_queue);
9887 free_tmf_tag_set:
9888 blk_mq_free_tag_set(&hba->tmf_tag_set);
9889 out_remove_scsi_host:
9890 scsi_remove_host(hba->host);
9891 out_disable:
9892 hba->is_irq_enabled = false;
9893 ufshcd_hba_exit(hba);
9894 out_error:
9895 return err;
9896 }
9897 EXPORT_SYMBOL_GPL(ufshcd_init);
9898
ufshcd_resume_complete(struct device * dev)9899 void ufshcd_resume_complete(struct device *dev)
9900 {
9901 struct ufs_hba *hba = dev_get_drvdata(dev);
9902
9903 if (hba->complete_put) {
9904 ufshcd_rpm_put(hba);
9905 hba->complete_put = false;
9906 }
9907 }
9908 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
9909
ufshcd_rpm_ok_for_spm(struct ufs_hba * hba)9910 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
9911 {
9912 struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
9913 enum ufs_dev_pwr_mode dev_pwr_mode;
9914 enum uic_link_state link_state;
9915 unsigned long flags;
9916 bool res;
9917
9918 spin_lock_irqsave(&dev->power.lock, flags);
9919 dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
9920 link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
9921 res = pm_runtime_suspended(dev) &&
9922 hba->curr_dev_pwr_mode == dev_pwr_mode &&
9923 hba->uic_link_state == link_state &&
9924 !hba->dev_info.b_rpm_dev_flush_capable;
9925 spin_unlock_irqrestore(&dev->power.lock, flags);
9926
9927 return res;
9928 }
9929
__ufshcd_suspend_prepare(struct device * dev,bool rpm_ok_for_spm)9930 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
9931 {
9932 struct ufs_hba *hba = dev_get_drvdata(dev);
9933 int ret;
9934
9935 /*
9936 * SCSI assumes that runtime-pm and system-pm for scsi drivers
9937 * are same. And it doesn't wake up the device for system-suspend
9938 * if it's runtime suspended. But ufs doesn't follow that.
9939 * Refer ufshcd_resume_complete()
9940 */
9941 if (hba->ufs_device_wlun) {
9942 /* Prevent runtime suspend */
9943 ufshcd_rpm_get_noresume(hba);
9944 /*
9945 * Check if already runtime suspended in same state as system
9946 * suspend would be.
9947 */
9948 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
9949 /* RPM state is not ok for SPM, so runtime resume */
9950 ret = ufshcd_rpm_resume(hba);
9951 if (ret < 0 && ret != -EACCES) {
9952 ufshcd_rpm_put(hba);
9953 return ret;
9954 }
9955 }
9956 hba->complete_put = true;
9957 }
9958 return 0;
9959 }
9960 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
9961
ufshcd_suspend_prepare(struct device * dev)9962 int ufshcd_suspend_prepare(struct device *dev)
9963 {
9964 return __ufshcd_suspend_prepare(dev, true);
9965 }
9966 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
9967
9968 #ifdef CONFIG_PM_SLEEP
ufshcd_wl_poweroff(struct device * dev)9969 static int ufshcd_wl_poweroff(struct device *dev)
9970 {
9971 struct scsi_device *sdev = to_scsi_device(dev);
9972 struct ufs_hba *hba = shost_priv(sdev->host);
9973
9974 __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9975 return 0;
9976 }
9977 #endif
9978
ufshcd_wl_probe(struct device * dev)9979 static int ufshcd_wl_probe(struct device *dev)
9980 {
9981 struct scsi_device *sdev = to_scsi_device(dev);
9982
9983 if (!is_device_wlun(sdev))
9984 return -ENODEV;
9985
9986 blk_pm_runtime_init(sdev->request_queue, dev);
9987 pm_runtime_set_autosuspend_delay(dev, 0);
9988 pm_runtime_allow(dev);
9989
9990 return 0;
9991 }
9992
ufshcd_wl_remove(struct device * dev)9993 static int ufshcd_wl_remove(struct device *dev)
9994 {
9995 pm_runtime_forbid(dev);
9996 return 0;
9997 }
9998
9999 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
10000 #ifdef CONFIG_PM_SLEEP
10001 .suspend = ufshcd_wl_suspend,
10002 .resume = ufshcd_wl_resume,
10003 .freeze = ufshcd_wl_suspend,
10004 .thaw = ufshcd_wl_resume,
10005 .poweroff = ufshcd_wl_poweroff,
10006 .restore = ufshcd_wl_resume,
10007 #endif
10008 SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
10009 };
10010
10011 /*
10012 * ufs_dev_wlun_template - describes ufs device wlun
10013 * ufs-device wlun - used to send pm commands
10014 * All luns are consumers of ufs-device wlun.
10015 *
10016 * Currently, no sd driver is present for wluns.
10017 * Hence the no specific pm operations are performed.
10018 * With ufs design, SSU should be sent to ufs-device wlun.
10019 * Hence register a scsi driver for ufs wluns only.
10020 */
10021 static struct scsi_driver ufs_dev_wlun_template = {
10022 .gendrv = {
10023 .name = "ufs_device_wlun",
10024 .owner = THIS_MODULE,
10025 .probe = ufshcd_wl_probe,
10026 .remove = ufshcd_wl_remove,
10027 .pm = &ufshcd_wl_pm_ops,
10028 .shutdown = ufshcd_wl_shutdown,
10029 },
10030 };
10031
ufshcd_core_init(void)10032 static int __init ufshcd_core_init(void)
10033 {
10034 int ret;
10035
10036 /* Verify that there are no gaps in struct utp_transfer_cmd_desc. */
10037 static_assert(sizeof(struct utp_transfer_cmd_desc) ==
10038 2 * ALIGNED_UPIU_SIZE +
10039 SG_ALL * sizeof(struct ufshcd_sg_entry));
10040
10041 ufs_debugfs_init();
10042
10043 ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
10044 if (ret)
10045 ufs_debugfs_exit();
10046 return ret;
10047 }
10048
ufshcd_core_exit(void)10049 static void __exit ufshcd_core_exit(void)
10050 {
10051 ufs_debugfs_exit();
10052 scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
10053 }
10054
10055 module_init(ufshcd_core_init);
10056 module_exit(ufshcd_core_exit);
10057
10058 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
10059 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
10060 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
10061 MODULE_LICENSE("GPL");
10062