1 /* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
2  *
3  * Redistribution and use in source and binary forms, with or without
4  * modification, are permitted provided that the following conditions are met:
5  *     * Redistributions of source code must retain the above copyright
6  *	 notice, this list of conditions and the following disclaimer.
7  *     * Redistributions in binary form must reproduce the above copyright
8  *	 notice, this list of conditions and the following disclaimer in the
9  *	 documentation and/or other materials provided with the distribution.
10  *     * Neither the name of Freescale Semiconductor nor the
11  *	 names of its contributors may be used to endorse or promote products
12  *	 derived from this software without specific prior written permission.
13  *
14  * ALTERNATIVELY, this software may be distributed under the terms of the
15  * GNU General Public License ("GPL") as published by the Free Software
16  * Foundation, either version 2 of that License or (at your option) any
17  * later version.
18  *
19  * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22  * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 #include "qman_priv.h"
32 
33 u16 qman_ip_rev;
34 EXPORT_SYMBOL(qman_ip_rev);
35 u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1;
36 EXPORT_SYMBOL(qm_channel_pool1);
37 u16 qm_channel_caam = QMAN_CHANNEL_CAAM;
38 EXPORT_SYMBOL(qm_channel_caam);
39 
40 /* Register offsets */
41 #define REG_QCSP_LIO_CFG(n)	(0x0000 + ((n) * 0x10))
42 #define REG_QCSP_IO_CFG(n)	(0x0004 + ((n) * 0x10))
43 #define REG_QCSP_DD_CFG(n)	(0x000c + ((n) * 0x10))
44 #define REG_DD_CFG		0x0200
45 #define REG_DCP_CFG(n)		(0x0300 + ((n) * 0x10))
46 #define REG_DCP_DD_CFG(n)	(0x0304 + ((n) * 0x10))
47 #define REG_DCP_DLM_AVG(n)	(0x030c + ((n) * 0x10))
48 #define REG_PFDR_FPC		0x0400
49 #define REG_PFDR_FP_HEAD	0x0404
50 #define REG_PFDR_FP_TAIL	0x0408
51 #define REG_PFDR_FP_LWIT	0x0410
52 #define REG_PFDR_CFG		0x0414
53 #define REG_SFDR_CFG		0x0500
54 #define REG_SFDR_IN_USE		0x0504
55 #define REG_WQ_CS_CFG(n)	(0x0600 + ((n) * 0x04))
56 #define REG_WQ_DEF_ENC_WQID	0x0630
57 #define REG_WQ_SC_DD_CFG(n)	(0x640 + ((n) * 0x04))
58 #define REG_WQ_PC_DD_CFG(n)	(0x680 + ((n) * 0x04))
59 #define REG_WQ_DC0_DD_CFG(n)	(0x6c0 + ((n) * 0x04))
60 #define REG_WQ_DC1_DD_CFG(n)	(0x700 + ((n) * 0x04))
61 #define REG_WQ_DCn_DD_CFG(n)	(0x6c0 + ((n) * 0x40)) /* n=2,3 */
62 #define REG_CM_CFG		0x0800
63 #define REG_ECSR		0x0a00
64 #define REG_ECIR		0x0a04
65 #define REG_EADR		0x0a08
66 #define REG_ECIR2		0x0a0c
67 #define REG_EDATA(n)		(0x0a10 + ((n) * 0x04))
68 #define REG_SBEC(n)		(0x0a80 + ((n) * 0x04))
69 #define REG_MCR			0x0b00
70 #define REG_MCP(n)		(0x0b04 + ((n) * 0x04))
71 #define REG_MISC_CFG		0x0be0
72 #define REG_HID_CFG		0x0bf0
73 #define REG_IDLE_STAT		0x0bf4
74 #define REG_IP_REV_1		0x0bf8
75 #define REG_IP_REV_2		0x0bfc
76 #define REG_FQD_BARE		0x0c00
77 #define REG_PFDR_BARE		0x0c20
78 #define REG_offset_BAR		0x0004	/* relative to REG_[FQD|PFDR]_BARE */
79 #define REG_offset_AR		0x0010	/* relative to REG_[FQD|PFDR]_BARE */
80 #define REG_QCSP_BARE		0x0c80
81 #define REG_QCSP_BAR		0x0c84
82 #define REG_CI_SCHED_CFG	0x0d00
83 #define REG_SRCIDR		0x0d04
84 #define REG_LIODNR		0x0d08
85 #define REG_CI_RLM_AVG		0x0d14
86 #define REG_ERR_ISR		0x0e00
87 #define REG_ERR_IER		0x0e04
88 #define REG_REV3_QCSP_LIO_CFG(n)	(0x1000 + ((n) * 0x10))
89 #define REG_REV3_QCSP_IO_CFG(n)	(0x1004 + ((n) * 0x10))
90 #define REG_REV3_QCSP_DD_CFG(n)	(0x100c + ((n) * 0x10))
91 
92 /* Assists for QMAN_MCR */
93 #define MCR_INIT_PFDR		0x01000000
94 #define MCR_get_rslt(v)		(u8)((v) >> 24)
95 #define MCR_rslt_idle(r)	(!(r) || ((r) >= 0xf0))
96 #define MCR_rslt_ok(r)		((r) == 0xf0)
97 #define MCR_rslt_eaccess(r)	((r) == 0xf8)
98 #define MCR_rslt_inval(r)	((r) == 0xff)
99 
100 /*
101  * Corenet initiator settings. Stash request queues are 4-deep to match cores
102  * ability to snarf. Stash priority is 3, other priorities are 2.
103  */
104 #define QM_CI_SCHED_CFG_SRCCIV		4
105 #define QM_CI_SCHED_CFG_SRQ_W		3
106 #define QM_CI_SCHED_CFG_RW_W		2
107 #define QM_CI_SCHED_CFG_BMAN_W		2
108 /* write SRCCIV enable */
109 #define QM_CI_SCHED_CFG_SRCCIV_EN	BIT(31)
110 
111 /* Follows WQ_CS_CFG0-5 */
112 enum qm_wq_class {
113 	qm_wq_portal = 0,
114 	qm_wq_pool = 1,
115 	qm_wq_fman0 = 2,
116 	qm_wq_fman1 = 3,
117 	qm_wq_caam = 4,
118 	qm_wq_pme = 5,
119 	qm_wq_first = qm_wq_portal,
120 	qm_wq_last = qm_wq_pme
121 };
122 
123 /* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */
124 enum qm_memory {
125 	qm_memory_fqd,
126 	qm_memory_pfdr
127 };
128 
129 /* Used by all error interrupt registers except 'inhibit' */
130 #define QM_EIRQ_CIDE	0x20000000	/* Corenet Initiator Data Error */
131 #define QM_EIRQ_CTDE	0x10000000	/* Corenet Target Data Error */
132 #define QM_EIRQ_CITT	0x08000000	/* Corenet Invalid Target Transaction */
133 #define QM_EIRQ_PLWI	0x04000000	/* PFDR Low Watermark */
134 #define QM_EIRQ_MBEI	0x02000000	/* Multi-bit ECC Error */
135 #define QM_EIRQ_SBEI	0x01000000	/* Single-bit ECC Error */
136 #define QM_EIRQ_PEBI	0x00800000	/* PFDR Enqueues Blocked Interrupt */
137 #define QM_EIRQ_IFSI	0x00020000	/* Invalid FQ Flow Control State */
138 #define QM_EIRQ_ICVI	0x00010000	/* Invalid Command Verb */
139 #define QM_EIRQ_IDDI	0x00000800	/* Invalid Dequeue (Direct-connect) */
140 #define QM_EIRQ_IDFI	0x00000400	/* Invalid Dequeue FQ */
141 #define QM_EIRQ_IDSI	0x00000200	/* Invalid Dequeue Source */
142 #define QM_EIRQ_IDQI	0x00000100	/* Invalid Dequeue Queue */
143 #define QM_EIRQ_IECE	0x00000010	/* Invalid Enqueue Configuration */
144 #define QM_EIRQ_IEOI	0x00000008	/* Invalid Enqueue Overflow */
145 #define QM_EIRQ_IESI	0x00000004	/* Invalid Enqueue State */
146 #define QM_EIRQ_IECI	0x00000002	/* Invalid Enqueue Channel */
147 #define QM_EIRQ_IEQI	0x00000001	/* Invalid Enqueue Queue */
148 
149 /* QMAN_ECIR valid error bit */
150 #define PORTAL_ECSR_ERR	(QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \
151 			 QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \
152 			 QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI)
153 #define FQID_ECSR_ERR	(QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \
154 			 QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \
155 			 QM_EIRQ_IFSI)
156 
157 struct qm_ecir {
158 	u32 info; /* res[30-31], ptyp[29], pnum[24-28], fqid[0-23] */
159 };
160 
qm_ecir_is_dcp(const struct qm_ecir * p)161 static bool qm_ecir_is_dcp(const struct qm_ecir *p)
162 {
163 	return p->info & BIT(29);
164 }
165 
qm_ecir_get_pnum(const struct qm_ecir * p)166 static int qm_ecir_get_pnum(const struct qm_ecir *p)
167 {
168 	return (p->info >> 24) & 0x1f;
169 }
170 
qm_ecir_get_fqid(const struct qm_ecir * p)171 static int qm_ecir_get_fqid(const struct qm_ecir *p)
172 {
173 	return p->info & (BIT(24) - 1);
174 }
175 
176 struct qm_ecir2 {
177 	u32 info; /* ptyp[31], res[10-30], pnum[0-9] */
178 };
179 
qm_ecir2_is_dcp(const struct qm_ecir2 * p)180 static bool qm_ecir2_is_dcp(const struct qm_ecir2 *p)
181 {
182 	return p->info & BIT(31);
183 }
184 
qm_ecir2_get_pnum(const struct qm_ecir2 * p)185 static int qm_ecir2_get_pnum(const struct qm_ecir2 *p)
186 {
187 	return p->info & (BIT(10) - 1);
188 }
189 
190 struct qm_eadr {
191 	u32 info; /* memid[24-27], eadr[0-11] */
192 		  /* v3: memid[24-28], eadr[0-15] */
193 };
194 
qm_eadr_get_memid(const struct qm_eadr * p)195 static int qm_eadr_get_memid(const struct qm_eadr *p)
196 {
197 	return (p->info >> 24) & 0xf;
198 }
199 
qm_eadr_get_eadr(const struct qm_eadr * p)200 static int qm_eadr_get_eadr(const struct qm_eadr *p)
201 {
202 	return p->info & (BIT(12) - 1);
203 }
204 
qm_eadr_v3_get_memid(const struct qm_eadr * p)205 static int qm_eadr_v3_get_memid(const struct qm_eadr *p)
206 {
207 	return (p->info >> 24) & 0x1f;
208 }
209 
qm_eadr_v3_get_eadr(const struct qm_eadr * p)210 static int qm_eadr_v3_get_eadr(const struct qm_eadr *p)
211 {
212 	return p->info & (BIT(16) - 1);
213 }
214 
215 struct qman_hwerr_txt {
216 	u32 mask;
217 	const char *txt;
218 };
219 
220 
221 static const struct qman_hwerr_txt qman_hwerr_txts[] = {
222 	{ QM_EIRQ_CIDE, "Corenet Initiator Data Error" },
223 	{ QM_EIRQ_CTDE, "Corenet Target Data Error" },
224 	{ QM_EIRQ_CITT, "Corenet Invalid Target Transaction" },
225 	{ QM_EIRQ_PLWI, "PFDR Low Watermark" },
226 	{ QM_EIRQ_MBEI, "Multi-bit ECC Error" },
227 	{ QM_EIRQ_SBEI, "Single-bit ECC Error" },
228 	{ QM_EIRQ_PEBI, "PFDR Enqueues Blocked Interrupt" },
229 	{ QM_EIRQ_ICVI, "Invalid Command Verb" },
230 	{ QM_EIRQ_IFSI, "Invalid Flow Control State" },
231 	{ QM_EIRQ_IDDI, "Invalid Dequeue (Direct-connect)" },
232 	{ QM_EIRQ_IDFI, "Invalid Dequeue FQ" },
233 	{ QM_EIRQ_IDSI, "Invalid Dequeue Source" },
234 	{ QM_EIRQ_IDQI, "Invalid Dequeue Queue" },
235 	{ QM_EIRQ_IECE, "Invalid Enqueue Configuration" },
236 	{ QM_EIRQ_IEOI, "Invalid Enqueue Overflow" },
237 	{ QM_EIRQ_IESI, "Invalid Enqueue State" },
238 	{ QM_EIRQ_IECI, "Invalid Enqueue Channel" },
239 	{ QM_EIRQ_IEQI, "Invalid Enqueue Queue" },
240 };
241 
242 struct qman_error_info_mdata {
243 	u16 addr_mask;
244 	u16 bits;
245 	const char *txt;
246 };
247 
248 static const struct qman_error_info_mdata error_mdata[] = {
249 	{ 0x01FF, 24, "FQD cache tag memory 0" },
250 	{ 0x01FF, 24, "FQD cache tag memory 1" },
251 	{ 0x01FF, 24, "FQD cache tag memory 2" },
252 	{ 0x01FF, 24, "FQD cache tag memory 3" },
253 	{ 0x0FFF, 512, "FQD cache memory" },
254 	{ 0x07FF, 128, "SFDR memory" },
255 	{ 0x01FF, 72, "WQ context memory" },
256 	{ 0x00FF, 240, "CGR memory" },
257 	{ 0x00FF, 302, "Internal Order Restoration List memory" },
258 	{ 0x01FF, 256, "SW portal ring memory" },
259 };
260 
261 #define QMAN_ERRS_TO_DISABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI)
262 
263 /*
264  * TODO: unimplemented registers
265  *
266  * Keeping a list here of QMan registers I have not yet covered;
267  * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR,
268  * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG,
269  * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12
270  */
271 
272 /* Pointer to the start of the QMan's CCSR space */
273 static u32 __iomem *qm_ccsr_start;
274 /* A SDQCR mask comprising all the available/visible pool channels */
275 static u32 qm_pools_sdqcr;
276 static int __qman_probed;
277 static int  __qman_requires_cleanup;
278 
qm_ccsr_in(u32 offset)279 static inline u32 qm_ccsr_in(u32 offset)
280 {
281 	return ioread32be(qm_ccsr_start + offset/4);
282 }
283 
qm_ccsr_out(u32 offset,u32 val)284 static inline void qm_ccsr_out(u32 offset, u32 val)
285 {
286 	iowrite32be(val, qm_ccsr_start + offset/4);
287 }
288 
qm_get_pools_sdqcr(void)289 u32 qm_get_pools_sdqcr(void)
290 {
291 	return qm_pools_sdqcr;
292 }
293 
294 enum qm_dc_portal {
295 	qm_dc_portal_fman0 = 0,
296 	qm_dc_portal_fman1 = 1
297 };
298 
qm_set_dc(enum qm_dc_portal portal,int ed,u8 sernd)299 static void qm_set_dc(enum qm_dc_portal portal, int ed, u8 sernd)
300 {
301 	DPAA_ASSERT(!ed || portal == qm_dc_portal_fman0 ||
302 		    portal == qm_dc_portal_fman1);
303 	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
304 		qm_ccsr_out(REG_DCP_CFG(portal),
305 			    (ed ? 0x1000 : 0) | (sernd & 0x3ff));
306 	else
307 		qm_ccsr_out(REG_DCP_CFG(portal),
308 			    (ed ? 0x100 : 0) | (sernd & 0x1f));
309 }
310 
qm_set_wq_scheduling(enum qm_wq_class wq_class,u8 cs_elev,u8 csw2,u8 csw3,u8 csw4,u8 csw5,u8 csw6,u8 csw7)311 static void qm_set_wq_scheduling(enum qm_wq_class wq_class,
312 				 u8 cs_elev, u8 csw2, u8 csw3, u8 csw4,
313 				 u8 csw5, u8 csw6, u8 csw7)
314 {
315 	qm_ccsr_out(REG_WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) |
316 		    ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) |
317 		    ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) |
318 		    ((csw6 & 0x7) << 4) | (csw7 & 0x7));
319 }
320 
qm_set_hid(void)321 static void qm_set_hid(void)
322 {
323 	qm_ccsr_out(REG_HID_CFG, 0);
324 }
325 
qm_set_corenet_initiator(void)326 static void qm_set_corenet_initiator(void)
327 {
328 	qm_ccsr_out(REG_CI_SCHED_CFG, QM_CI_SCHED_CFG_SRCCIV_EN |
329 		    (QM_CI_SCHED_CFG_SRCCIV << 24) |
330 		    (QM_CI_SCHED_CFG_SRQ_W << 8) |
331 		    (QM_CI_SCHED_CFG_RW_W << 4) |
332 		    QM_CI_SCHED_CFG_BMAN_W);
333 }
334 
qm_get_version(u16 * id,u8 * major,u8 * minor)335 static void qm_get_version(u16 *id, u8 *major, u8 *minor)
336 {
337 	u32 v = qm_ccsr_in(REG_IP_REV_1);
338 	*id = (v >> 16);
339 	*major = (v >> 8) & 0xff;
340 	*minor = v & 0xff;
341 }
342 
343 #define PFDR_AR_EN		BIT(31)
qm_set_memory(enum qm_memory memory,u64 ba,u32 size)344 static int qm_set_memory(enum qm_memory memory, u64 ba, u32 size)
345 {
346 	void *ptr;
347 	u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE;
348 	u32 exp = ilog2(size);
349 	u32 bar, bare;
350 
351 	/* choke if size isn't within range */
352 	DPAA_ASSERT((size >= 4096) && (size <= 1024*1024*1024) &&
353 		    is_power_of_2(size));
354 	/* choke if 'ba' has lower-alignment than 'size' */
355 	DPAA_ASSERT(!(ba & (size - 1)));
356 
357 	/* Check to see if QMan has already been initialized */
358 	bar = qm_ccsr_in(offset + REG_offset_BAR);
359 	if (bar) {
360 		/* Maker sure ba == what was programmed) */
361 		bare = qm_ccsr_in(offset);
362 		if (bare != upper_32_bits(ba) || bar != lower_32_bits(ba)) {
363 			pr_err("Attempted to reinitialize QMan with different BAR, got 0x%llx read BARE=0x%x BAR=0x%x\n",
364 			       ba, bare, bar);
365 			return -ENOMEM;
366 		}
367 		__qman_requires_cleanup = 1;
368 		/* Return 1 to indicate memory was previously programmed */
369 		return 1;
370 	}
371 	/* Need to temporarily map the area to make sure it is zeroed */
372 	ptr = memremap(ba, size, MEMREMAP_WB);
373 	if (!ptr) {
374 		pr_crit("memremap() of QMan private memory failed\n");
375 		return -ENOMEM;
376 	}
377 	memset(ptr, 0, size);
378 
379 #ifdef CONFIG_PPC
380 	/*
381 	 * PPC doesn't appear to flush the cache on memunmap() but the
382 	 * cache must be flushed since QMan does non coherent accesses
383 	 * to this memory
384 	 */
385 	flush_dcache_range((unsigned long) ptr, (unsigned long) ptr+size);
386 #endif
387 	memunmap(ptr);
388 
389 	qm_ccsr_out(offset, upper_32_bits(ba));
390 	qm_ccsr_out(offset + REG_offset_BAR, lower_32_bits(ba));
391 	qm_ccsr_out(offset + REG_offset_AR, PFDR_AR_EN | (exp - 1));
392 	return 0;
393 }
394 
qm_set_pfdr_threshold(u32 th,u8 k)395 static void qm_set_pfdr_threshold(u32 th, u8 k)
396 {
397 	qm_ccsr_out(REG_PFDR_FP_LWIT, th & 0xffffff);
398 	qm_ccsr_out(REG_PFDR_CFG, k);
399 }
400 
qm_set_sfdr_threshold(u16 th)401 static void qm_set_sfdr_threshold(u16 th)
402 {
403 	qm_ccsr_out(REG_SFDR_CFG, th & 0x3ff);
404 }
405 
qm_init_pfdr(struct device * dev,u32 pfdr_start,u32 num)406 static int qm_init_pfdr(struct device *dev, u32 pfdr_start, u32 num)
407 {
408 	u8 rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR));
409 
410 	DPAA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num);
411 	/* Make sure the command interface is 'idle' */
412 	if (!MCR_rslt_idle(rslt)) {
413 		dev_crit(dev, "QMAN_MCR isn't idle");
414 		WARN_ON(1);
415 	}
416 
417 	/* Write the MCR command params then the verb */
418 	qm_ccsr_out(REG_MCP(0), pfdr_start);
419 	/*
420 	 * TODO: remove this - it's a workaround for a model bug that is
421 	 * corrected in more recent versions. We use the workaround until
422 	 * everyone has upgraded.
423 	 */
424 	qm_ccsr_out(REG_MCP(1), pfdr_start + num - 16);
425 	dma_wmb();
426 	qm_ccsr_out(REG_MCR, MCR_INIT_PFDR);
427 	/* Poll for the result */
428 	do {
429 		rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR));
430 	} while (!MCR_rslt_idle(rslt));
431 	if (MCR_rslt_ok(rslt))
432 		return 0;
433 	if (MCR_rslt_eaccess(rslt))
434 		return -EACCES;
435 	if (MCR_rslt_inval(rslt))
436 		return -EINVAL;
437 	dev_crit(dev, "Unexpected result from MCR_INIT_PFDR: %02x\n", rslt);
438 	return -ENODEV;
439 }
440 
441 /*
442  * QMan needs two global memory areas initialized at boot time:
443  *  1) FQD: Frame Queue Descriptors used to manage frame queues
444  *  2) PFDR: Packed Frame Queue Descriptor Records used to store frames
445  * Both areas are reserved using the device tree reserved memory framework
446  * and the addresses and sizes are initialized when the QMan device is probed
447  */
448 static dma_addr_t fqd_a, pfdr_a;
449 static size_t fqd_sz, pfdr_sz;
450 
451 #ifdef CONFIG_PPC
452 /*
453  * Support for PPC Device Tree backward compatibility when compatible
454  * string is set to fsl-qman-fqd and fsl-qman-pfdr
455  */
zero_priv_mem(phys_addr_t addr,size_t sz)456 static int zero_priv_mem(phys_addr_t addr, size_t sz)
457 {
458 	/* map as cacheable, non-guarded */
459 	void __iomem *tmpp = ioremap_cache(addr, sz);
460 
461 	if (!tmpp)
462 		return -ENOMEM;
463 
464 	memset_io(tmpp, 0, sz);
465 	flush_dcache_range((unsigned long)tmpp,
466 			   (unsigned long)tmpp + sz);
467 	iounmap(tmpp);
468 
469 	return 0;
470 }
471 
qman_fqd(struct reserved_mem * rmem)472 static int qman_fqd(struct reserved_mem *rmem)
473 {
474 	fqd_a = rmem->base;
475 	fqd_sz = rmem->size;
476 
477 	WARN_ON(!(fqd_a && fqd_sz));
478 	return 0;
479 }
480 RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd);
481 
qman_pfdr(struct reserved_mem * rmem)482 static int qman_pfdr(struct reserved_mem *rmem)
483 {
484 	pfdr_a = rmem->base;
485 	pfdr_sz = rmem->size;
486 
487 	WARN_ON(!(pfdr_a && pfdr_sz));
488 
489 	return 0;
490 }
491 RESERVEDMEM_OF_DECLARE(qman_pfdr, "fsl,qman-pfdr", qman_pfdr);
492 
493 #endif
494 
qm_get_fqid_maxcnt(void)495 unsigned int qm_get_fqid_maxcnt(void)
496 {
497 	return fqd_sz / 64;
498 }
499 
log_edata_bits(struct device * dev,u32 bit_count)500 static void log_edata_bits(struct device *dev, u32 bit_count)
501 {
502 	u32 i, j, mask = 0xffffffff;
503 
504 	dev_warn(dev, "ErrInt, EDATA:\n");
505 	i = bit_count / 32;
506 	if (bit_count % 32) {
507 		i++;
508 		mask = ~(mask << bit_count % 32);
509 	}
510 	j = 16 - i;
511 	dev_warn(dev, "  0x%08x\n", qm_ccsr_in(REG_EDATA(j)) & mask);
512 	j++;
513 	for (; j < 16; j++)
514 		dev_warn(dev, "  0x%08x\n", qm_ccsr_in(REG_EDATA(j)));
515 }
516 
log_additional_error_info(struct device * dev,u32 isr_val,u32 ecsr_val)517 static void log_additional_error_info(struct device *dev, u32 isr_val,
518 				      u32 ecsr_val)
519 {
520 	struct qm_ecir ecir_val;
521 	struct qm_eadr eadr_val;
522 	int memid;
523 
524 	ecir_val.info = qm_ccsr_in(REG_ECIR);
525 	/* Is portal info valid */
526 	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
527 		struct qm_ecir2 ecir2_val;
528 
529 		ecir2_val.info = qm_ccsr_in(REG_ECIR2);
530 		if (ecsr_val & PORTAL_ECSR_ERR) {
531 			dev_warn(dev, "ErrInt: %s id %d\n",
532 				 qm_ecir2_is_dcp(&ecir2_val) ? "DCP" : "SWP",
533 				 qm_ecir2_get_pnum(&ecir2_val));
534 		}
535 		if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE))
536 			dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n",
537 				 qm_ecir_get_fqid(&ecir_val));
538 
539 		if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
540 			eadr_val.info = qm_ccsr_in(REG_EADR);
541 			memid = qm_eadr_v3_get_memid(&eadr_val);
542 			dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n",
543 				 error_mdata[memid].txt,
544 				 error_mdata[memid].addr_mask
545 					& qm_eadr_v3_get_eadr(&eadr_val));
546 			log_edata_bits(dev, error_mdata[memid].bits);
547 		}
548 	} else {
549 		if (ecsr_val & PORTAL_ECSR_ERR) {
550 			dev_warn(dev, "ErrInt: %s id %d\n",
551 				 qm_ecir_is_dcp(&ecir_val) ? "DCP" : "SWP",
552 				 qm_ecir_get_pnum(&ecir_val));
553 		}
554 		if (ecsr_val & FQID_ECSR_ERR)
555 			dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n",
556 				 qm_ecir_get_fqid(&ecir_val));
557 
558 		if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
559 			eadr_val.info = qm_ccsr_in(REG_EADR);
560 			memid = qm_eadr_get_memid(&eadr_val);
561 			dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n",
562 				 error_mdata[memid].txt,
563 				 error_mdata[memid].addr_mask
564 					& qm_eadr_get_eadr(&eadr_val));
565 			log_edata_bits(dev, error_mdata[memid].bits);
566 		}
567 	}
568 }
569 
qman_isr(int irq,void * ptr)570 static irqreturn_t qman_isr(int irq, void *ptr)
571 {
572 	u32 isr_val, ier_val, ecsr_val, isr_mask, i;
573 	struct device *dev = ptr;
574 
575 	ier_val = qm_ccsr_in(REG_ERR_IER);
576 	isr_val = qm_ccsr_in(REG_ERR_ISR);
577 	ecsr_val = qm_ccsr_in(REG_ECSR);
578 	isr_mask = isr_val & ier_val;
579 
580 	if (!isr_mask)
581 		return IRQ_NONE;
582 
583 	for (i = 0; i < ARRAY_SIZE(qman_hwerr_txts); i++) {
584 		if (qman_hwerr_txts[i].mask & isr_mask) {
585 			dev_err_ratelimited(dev, "ErrInt: %s\n",
586 					    qman_hwerr_txts[i].txt);
587 			if (qman_hwerr_txts[i].mask & ecsr_val) {
588 				log_additional_error_info(dev, isr_mask,
589 							  ecsr_val);
590 				/* Re-arm error capture registers */
591 				qm_ccsr_out(REG_ECSR, ecsr_val);
592 			}
593 			if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_DISABLE) {
594 				dev_dbg(dev, "Disabling error 0x%x\n",
595 					qman_hwerr_txts[i].mask);
596 				ier_val &= ~qman_hwerr_txts[i].mask;
597 				qm_ccsr_out(REG_ERR_IER, ier_val);
598 			}
599 		}
600 	}
601 	qm_ccsr_out(REG_ERR_ISR, isr_val);
602 
603 	return IRQ_HANDLED;
604 }
605 
qman_init_ccsr(struct device * dev)606 static int qman_init_ccsr(struct device *dev)
607 {
608 	int i, err;
609 
610 	/* FQD memory */
611 	err = qm_set_memory(qm_memory_fqd, fqd_a, fqd_sz);
612 	if (err < 0)
613 		return err;
614 	/* PFDR memory */
615 	err = qm_set_memory(qm_memory_pfdr, pfdr_a, pfdr_sz);
616 	if (err < 0)
617 		return err;
618 	/* Only initialize PFDRs if the QMan was not initialized before */
619 	if (err == 0) {
620 		err = qm_init_pfdr(dev, 8, pfdr_sz / 64 - 8);
621 		if (err)
622 			return err;
623 	}
624 	/* thresholds */
625 	qm_set_pfdr_threshold(512, 64);
626 	qm_set_sfdr_threshold(128);
627 	/* clear stale PEBI bit from interrupt status register */
628 	qm_ccsr_out(REG_ERR_ISR, QM_EIRQ_PEBI);
629 	/* corenet initiator settings */
630 	qm_set_corenet_initiator();
631 	/* HID settings */
632 	qm_set_hid();
633 	/* Set scheduling weights to defaults */
634 	for (i = qm_wq_first; i <= qm_wq_last; i++)
635 		qm_set_wq_scheduling(i, 0, 0, 0, 0, 0, 0, 0);
636 	/* We are not prepared to accept ERNs for hardware enqueues */
637 	qm_set_dc(qm_dc_portal_fman0, 1, 0);
638 	qm_set_dc(qm_dc_portal_fman1, 1, 0);
639 	return 0;
640 }
641 
642 #define LIO_CFG_LIODN_MASK 0x0fff0000
__qman_liodn_fixup(u16 channel)643 void __qman_liodn_fixup(u16 channel)
644 {
645 	static int done;
646 	static u32 liodn_offset;
647 	u32 before, after;
648 	int idx = channel - QM_CHANNEL_SWPORTAL0;
649 
650 	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
651 		before = qm_ccsr_in(REG_REV3_QCSP_LIO_CFG(idx));
652 	else
653 		before = qm_ccsr_in(REG_QCSP_LIO_CFG(idx));
654 	if (!done) {
655 		liodn_offset = before & LIO_CFG_LIODN_MASK;
656 		done = 1;
657 		return;
658 	}
659 	after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset;
660 	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
661 		qm_ccsr_out(REG_REV3_QCSP_LIO_CFG(idx), after);
662 	else
663 		qm_ccsr_out(REG_QCSP_LIO_CFG(idx), after);
664 }
665 
666 #define IO_CFG_SDEST_MASK 0x00ff0000
qman_set_sdest(u16 channel,unsigned int cpu_idx)667 void qman_set_sdest(u16 channel, unsigned int cpu_idx)
668 {
669 	int idx = channel - QM_CHANNEL_SWPORTAL0;
670 	u32 before, after;
671 
672 	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
673 		before = qm_ccsr_in(REG_REV3_QCSP_IO_CFG(idx));
674 		/* Each pair of vcpu share the same SRQ(SDEST) */
675 		cpu_idx /= 2;
676 		after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
677 		qm_ccsr_out(REG_REV3_QCSP_IO_CFG(idx), after);
678 	} else {
679 		before = qm_ccsr_in(REG_QCSP_IO_CFG(idx));
680 		after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
681 		qm_ccsr_out(REG_QCSP_IO_CFG(idx), after);
682 	}
683 }
684 
qman_resource_init(struct device * dev)685 static int qman_resource_init(struct device *dev)
686 {
687 	int pool_chan_num, cgrid_num;
688 	int ret, i;
689 
690 	switch (qman_ip_rev >> 8) {
691 	case 1:
692 		pool_chan_num = 15;
693 		cgrid_num = 256;
694 		break;
695 	case 2:
696 		pool_chan_num = 3;
697 		cgrid_num = 64;
698 		break;
699 	case 3:
700 		pool_chan_num = 15;
701 		cgrid_num = 256;
702 		break;
703 	default:
704 		return -ENODEV;
705 	}
706 
707 	ret = gen_pool_add(qm_qpalloc, qm_channel_pool1 | DPAA_GENALLOC_OFF,
708 			   pool_chan_num, -1);
709 	if (ret) {
710 		dev_err(dev, "Failed to seed pool channels (%d)\n", ret);
711 		return ret;
712 	}
713 
714 	ret = gen_pool_add(qm_cgralloc, DPAA_GENALLOC_OFF, cgrid_num, -1);
715 	if (ret) {
716 		dev_err(dev, "Failed to seed CGRID range (%d)\n", ret);
717 		return ret;
718 	}
719 
720 	/* parse pool channels into the SDQCR mask */
721 	for (i = 0; i < cgrid_num; i++)
722 		qm_pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(i);
723 
724 	ret = gen_pool_add(qm_fqalloc, QM_FQID_RANGE_START | DPAA_GENALLOC_OFF,
725 			   qm_get_fqid_maxcnt() - QM_FQID_RANGE_START, -1);
726 	if (ret) {
727 		dev_err(dev, "Failed to seed FQID range (%d)\n", ret);
728 		return ret;
729 	}
730 
731 	return 0;
732 }
733 
qman_is_probed(void)734 int qman_is_probed(void)
735 {
736 	return __qman_probed;
737 }
738 EXPORT_SYMBOL_GPL(qman_is_probed);
739 
qman_requires_cleanup(void)740 int qman_requires_cleanup(void)
741 {
742 	return __qman_requires_cleanup;
743 }
744 
qman_done_cleanup(void)745 void qman_done_cleanup(void)
746 {
747 	qman_enable_irqs();
748 	__qman_requires_cleanup = 0;
749 }
750 
751 
fsl_qman_probe(struct platform_device * pdev)752 static int fsl_qman_probe(struct platform_device *pdev)
753 {
754 	struct device *dev = &pdev->dev;
755 	struct device_node *node = dev->of_node;
756 	struct resource *res;
757 	int ret, err_irq;
758 	u16 id;
759 	u8 major, minor;
760 
761 	__qman_probed = -1;
762 
763 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
764 	if (!res) {
765 		dev_err(dev, "Can't get %pOF property 'IORESOURCE_MEM'\n",
766 			node);
767 		return -ENXIO;
768 	}
769 	qm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res));
770 	if (!qm_ccsr_start)
771 		return -ENXIO;
772 
773 	qm_get_version(&id, &major, &minor);
774 	if (major == 1 && minor == 0) {
775 		dev_err(dev, "Rev1.0 on P4080 rev1 is not supported!\n");
776 			return -ENODEV;
777 	} else if (major == 1 && minor == 1)
778 		qman_ip_rev = QMAN_REV11;
779 	else if	(major == 1 && minor == 2)
780 		qman_ip_rev = QMAN_REV12;
781 	else if (major == 2 && minor == 0)
782 		qman_ip_rev = QMAN_REV20;
783 	else if (major == 3 && minor == 0)
784 		qman_ip_rev = QMAN_REV30;
785 	else if (major == 3 && minor == 1)
786 		qman_ip_rev = QMAN_REV31;
787 	else if (major == 3 && minor == 2)
788 		qman_ip_rev = QMAN_REV32;
789 	else {
790 		dev_err(dev, "Unknown QMan version\n");
791 		return -ENODEV;
792 	}
793 
794 	if ((qman_ip_rev & 0xff00) >= QMAN_REV30) {
795 		qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3;
796 		qm_channel_caam = QMAN_CHANNEL_CAAM_REV3;
797 	}
798 
799 	if (fqd_a) {
800 #ifdef CONFIG_PPC
801 		/*
802 		 * For PPC backward DT compatibility
803 		 * FQD memory MUST be zero'd by software
804 		 */
805 		zero_priv_mem(fqd_a, fqd_sz);
806 #else
807 		WARN(1, "Unexpected architecture using non shared-dma-mem reservations");
808 #endif
809 	} else {
810 		/*
811 		 * Order of memory regions is assumed as FQD followed by PFDR
812 		 * in order to ensure allocations from the correct regions the
813 		 * driver initializes then allocates each piece in order
814 		 */
815 		ret = qbman_init_private_mem(dev, 0, &fqd_a, &fqd_sz);
816 		if (ret) {
817 			dev_err(dev, "qbman_init_private_mem() for FQD failed 0x%x\n",
818 				ret);
819 			return -ENODEV;
820 		}
821 	}
822 	dev_dbg(dev, "Allocated FQD 0x%llx 0x%zx\n", fqd_a, fqd_sz);
823 
824 	if (!pfdr_a) {
825 		/* Setup PFDR memory */
826 		ret = qbman_init_private_mem(dev, 1, &pfdr_a, &pfdr_sz);
827 		if (ret) {
828 			dev_err(dev, "qbman_init_private_mem() for PFDR failed 0x%x\n",
829 				ret);
830 			return -ENODEV;
831 		}
832 	}
833 	dev_dbg(dev, "Allocated PFDR 0x%llx 0x%zx\n", pfdr_a, pfdr_sz);
834 
835 	ret = qman_init_ccsr(dev);
836 	if (ret) {
837 		dev_err(dev, "CCSR setup failed\n");
838 		return ret;
839 	}
840 
841 	err_irq = platform_get_irq(pdev, 0);
842 	if (err_irq <= 0) {
843 		dev_info(dev, "Can't get %pOF property 'interrupts'\n",
844 			 node);
845 		return -ENODEV;
846 	}
847 	ret = devm_request_irq(dev, err_irq, qman_isr, IRQF_SHARED, "qman-err",
848 			       dev);
849 	if (ret)  {
850 		dev_err(dev, "devm_request_irq() failed %d for '%pOF'\n",
851 			ret, node);
852 		return ret;
853 	}
854 
855 	/*
856 	 * Write-to-clear any stale bits, (eg. starvation being asserted prior
857 	 * to resource allocation during driver init).
858 	 */
859 	qm_ccsr_out(REG_ERR_ISR, 0xffffffff);
860 	/* Enable Error Interrupts */
861 	qm_ccsr_out(REG_ERR_IER, 0xffffffff);
862 
863 	qm_fqalloc = devm_gen_pool_create(dev, 0, -1, "qman-fqalloc");
864 	if (IS_ERR(qm_fqalloc)) {
865 		ret = PTR_ERR(qm_fqalloc);
866 		dev_err(dev, "qman-fqalloc pool init failed (%d)\n", ret);
867 		return ret;
868 	}
869 
870 	qm_qpalloc = devm_gen_pool_create(dev, 0, -1, "qman-qpalloc");
871 	if (IS_ERR(qm_qpalloc)) {
872 		ret = PTR_ERR(qm_qpalloc);
873 		dev_err(dev, "qman-qpalloc pool init failed (%d)\n", ret);
874 		return ret;
875 	}
876 
877 	qm_cgralloc = devm_gen_pool_create(dev, 0, -1, "qman-cgralloc");
878 	if (IS_ERR(qm_cgralloc)) {
879 		ret = PTR_ERR(qm_cgralloc);
880 		dev_err(dev, "qman-cgralloc pool init failed (%d)\n", ret);
881 		return ret;
882 	}
883 
884 	ret = qman_resource_init(dev);
885 	if (ret)
886 		return ret;
887 
888 	ret = qman_alloc_fq_table(qm_get_fqid_maxcnt());
889 	if (ret)
890 		return ret;
891 
892 	ret = qman_wq_alloc();
893 	if (ret)
894 		return ret;
895 
896 	__qman_probed = 1;
897 
898 	return 0;
899 }
900 
901 static const struct of_device_id fsl_qman_ids[] = {
902 	{
903 		.compatible = "fsl,qman",
904 	},
905 	{}
906 };
907 
908 static struct platform_driver fsl_qman_driver = {
909 	.driver = {
910 		.name = KBUILD_MODNAME,
911 		.of_match_table = fsl_qman_ids,
912 		.suppress_bind_attrs = true,
913 	},
914 	.probe = fsl_qman_probe,
915 };
916 
917 builtin_platform_driver(fsl_qman_driver);
918