1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2018 Intel Corporation
3
4 #include <linux/device.h>
5 #include <linux/iopoll.h>
6 #include <linux/slab.h>
7
8 #include "ipu3.h"
9 #include "ipu3-css.h"
10 #include "ipu3-css-fw.h"
11 #include "ipu3-css-params.h"
12 #include "ipu3-dmamap.h"
13 #include "ipu3-tables.h"
14
15 /* IRQ configuration */
16 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \
17 IMGU_IRQCTRL_IRQ_SP2 | \
18 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
19 IMGU_IRQCTRL_IRQ_SW_PIN(1))
20
21 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */
22
23 /* Some sane limits for resolutions */
24 #define IPU3_CSS_MIN_RES 32
25 #define IPU3_CSS_MAX_H 3136
26 #define IPU3_CSS_MAX_W 4224
27
28 /* minimal envelope size(GDC in - out) should be 4 */
29 #define MIN_ENVELOPE 4
30
31 /*
32 * pre-allocated buffer size for CSS ABI, auxiliary frames
33 * after BDS and before GDC. Those values should be tuned
34 * to big enough to avoid buffer re-allocation when
35 * streaming to lower streaming latency.
36 */
37 #define CSS_ABI_SIZE 136
38 #define CSS_BDS_SIZE (4480 * 3200 * 3)
39 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8)
40
41 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q))
42 #define IPU3_CSS_FORMAT_FL_IN \
43 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
44 #define IPU3_CSS_FORMAT_FL_OUT \
45 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
46 #define IPU3_CSS_FORMAT_FL_VF \
47 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
48
49 /* Formats supported by IPU3 Camera Sub System */
50 static const struct imgu_css_format imgu_css_formats[] = {
51 {
52 .pixelformat = V4L2_PIX_FMT_NV12,
53 .colorspace = V4L2_COLORSPACE_SRGB,
54 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
55 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
56 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
57 .chroma_decim = 4,
58 .width_align = IPU3_UAPI_ISP_VEC_ELEMS,
59 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
60 }, {
61 /* Each 32 bytes contains 25 10-bit pixels */
62 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
63 .colorspace = V4L2_COLORSPACE_RAW,
64 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
65 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
66 .bit_depth = 10,
67 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
68 .flags = IPU3_CSS_FORMAT_FL_IN,
69 }, {
70 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
71 .colorspace = V4L2_COLORSPACE_RAW,
72 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
73 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
74 .bit_depth = 10,
75 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
76 .flags = IPU3_CSS_FORMAT_FL_IN,
77 }, {
78 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
79 .colorspace = V4L2_COLORSPACE_RAW,
80 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
81 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
82 .bit_depth = 10,
83 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
84 .flags = IPU3_CSS_FORMAT_FL_IN,
85 }, {
86 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
87 .colorspace = V4L2_COLORSPACE_RAW,
88 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
89 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
90 .bit_depth = 10,
91 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
92 .flags = IPU3_CSS_FORMAT_FL_IN,
93 },
94 };
95
96 static const struct {
97 enum imgu_abi_queue_id qid;
98 size_t ptr_ofs;
99 } imgu_css_queues[IPU3_CSS_QUEUES] = {
100 [IPU3_CSS_QUEUE_IN] = {
101 IMGU_ABI_QUEUE_C_ID,
102 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
103 },
104 [IPU3_CSS_QUEUE_OUT] = {
105 IMGU_ABI_QUEUE_D_ID,
106 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
107 },
108 [IPU3_CSS_QUEUE_VF] = {
109 IMGU_ABI_QUEUE_E_ID,
110 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
111 },
112 [IPU3_CSS_QUEUE_STAT_3A] = {
113 IMGU_ABI_QUEUE_F_ID,
114 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
115 },
116 };
117
118 /* Initialize queue based on given format, adjust format as needed */
imgu_css_queue_init(struct imgu_css_queue * queue,struct v4l2_pix_format_mplane * fmt,u32 flags)119 static int imgu_css_queue_init(struct imgu_css_queue *queue,
120 struct v4l2_pix_format_mplane *fmt, u32 flags)
121 {
122 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
123 unsigned int i;
124 u32 sizeimage;
125
126 INIT_LIST_HEAD(&queue->bufs);
127
128 queue->css_fmt = NULL; /* Disable */
129 if (!fmt)
130 return 0;
131
132 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
133 if (!(imgu_css_formats[i].flags & flags))
134 continue;
135 queue->css_fmt = &imgu_css_formats[i];
136 if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
137 break;
138 }
139 if (!queue->css_fmt)
140 return -EINVAL; /* Could not find any suitable format */
141
142 queue->fmt.mpix = *fmt;
143
144 f->width = ALIGN(clamp_t(u32, f->width,
145 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
146 f->height = ALIGN(clamp_t(u32, f->height,
147 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
148 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
149 f->plane_fmt[0].bytesperline =
150 imgu_bytesperline(f->width, queue->css_fmt->frame_format);
151 sizeimage = f->height * f->plane_fmt[0].bytesperline;
152 if (queue->css_fmt->chroma_decim)
153 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
154
155 f->plane_fmt[0].sizeimage = sizeimage;
156 f->field = V4L2_FIELD_NONE;
157 f->num_planes = 1;
158 f->colorspace = queue->css_fmt->colorspace;
159 f->flags = 0;
160 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
161 f->quantization = V4L2_QUANTIZATION_DEFAULT;
162 f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
163 memset(f->reserved, 0, sizeof(f->reserved));
164
165 return 0;
166 }
167
imgu_css_queue_enabled(struct imgu_css_queue * q)168 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
169 {
170 return q->css_fmt;
171 }
172
173 /******************* css hw *******************/
174
175 /* In the style of writesl() defined in include/asm-generic/io.h */
writes(const void * mem,ssize_t count,void __iomem * addr)176 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
177 {
178 if (count >= 4) {
179 const u32 *buf = mem;
180
181 count /= 4;
182 do {
183 writel(*buf++, addr);
184 addr += 4;
185 } while (--count);
186 }
187 }
188
189 /* Wait until register `reg', masked with `mask', becomes `cmp' */
imgu_hw_wait(void __iomem * base,int reg,u32 mask,u32 cmp)190 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
191 {
192 u32 val;
193
194 return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
195 1000, 100 * 1000);
196 }
197
198 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
199
imgu_css_set_powerup(struct device * dev,void __iomem * base,unsigned int freq)200 int imgu_css_set_powerup(struct device *dev, void __iomem *base,
201 unsigned int freq)
202 {
203 u32 pm_ctrl, state, val;
204
205 dev_dbg(dev, "%s with freq %u\n", __func__, freq);
206 /* Clear the CSS busy signal */
207 readl(base + IMGU_REG_GP_BUSY);
208 writel(0, base + IMGU_REG_GP_BUSY);
209
210 /* Wait for idle signal */
211 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
212 IMGU_STATE_IDLE_STS)) {
213 dev_err(dev, "failed to set CSS idle\n");
214 goto fail;
215 }
216
217 /* Reset the css */
218 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
219 base + IMGU_REG_PM_CTRL);
220
221 usleep_range(200, 300);
222
223 /** Prepare CSS */
224
225 pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
226 state = readl(base + IMGU_REG_STATE);
227
228 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
229 pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
230
231 /* Power up CSS using wrapper */
232 if (state & IMGU_STATE_POWER_DOWN) {
233 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
234 base + IMGU_REG_PM_CTRL);
235 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
236 IMGU_PM_CTRL_START, 0)) {
237 dev_err(dev, "failed to power up CSS\n");
238 goto fail;
239 }
240 usleep_range(2000, 3000);
241 } else {
242 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
243 }
244
245 /* Set the busy bit */
246 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
247
248 /* Set CSS clock frequency */
249 pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
250 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
251 writel(val, base + IMGU_REG_PM_CTRL);
252 writel(0, base + IMGU_REG_GP_BUSY);
253 if (imgu_hw_wait(base, IMGU_REG_STATE,
254 IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
255 dev_err(dev, "failed to pwrdn CSS\n");
256 goto fail;
257 }
258 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
259 writel(val, base + IMGU_REG_SYSTEM_REQ);
260 writel(1, base + IMGU_REG_GP_BUSY);
261 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
262 base + IMGU_REG_PM_CTRL);
263 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
264 IMGU_STATE_HALT_STS)) {
265 dev_err(dev, "failed to halt CSS\n");
266 goto fail;
267 }
268
269 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
270 base + IMGU_REG_PM_CTRL);
271 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
272 dev_err(dev, "failed to start CSS\n");
273 goto fail;
274 }
275 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
276 base + IMGU_REG_PM_CTRL);
277
278 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */
279 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
280 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
281 writel(val, base + IMGU_REG_PM_CTRL);
282
283 return 0;
284
285 fail:
286 imgu_css_set_powerdown(dev, base);
287 return -EIO;
288 }
289
imgu_css_set_powerdown(struct device * dev,void __iomem * base)290 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
291 {
292 dev_dbg(dev, "%s\n", __func__);
293 /* wait for cio idle signal */
294 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
295 IMGU_CIO_GATE_BURST_MASK, 0))
296 dev_warn(dev, "wait cio gate idle timeout");
297
298 /* wait for css idle signal */
299 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
300 IMGU_STATE_IDLE_STS))
301 dev_warn(dev, "wait css idle timeout\n");
302
303 /* do halt-halted handshake with css */
304 writel(1, base + IMGU_REG_GP_HALT);
305 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
306 IMGU_STATE_HALT_STS))
307 dev_warn(dev, "failed to halt css");
308
309 /* de-assert the busy bit */
310 writel(0, base + IMGU_REG_GP_BUSY);
311 }
312
imgu_css_hw_enable_irq(struct imgu_css * css)313 static void imgu_css_hw_enable_irq(struct imgu_css *css)
314 {
315 void __iomem *const base = css->base;
316 u32 val, i;
317
318 /* Set up interrupts */
319
320 /*
321 * Enable IRQ on the SP which signals that SP goes to idle
322 * (aka ready state) and set trigger to pulse
323 */
324 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
325 writel(val, base + IMGU_REG_SP_CTRL(0));
326 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
327
328 /* Enable IRQs from the IMGU wrapper */
329 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
330 /* Clear */
331 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
332
333 /* Enable IRQs from main IRQ controller */
334 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
335 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
336 writel(IMGU_IRQCTRL_IRQ_MASK,
337 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
338 writel(IMGU_IRQCTRL_IRQ_MASK,
339 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
340 writel(IMGU_IRQCTRL_IRQ_MASK,
341 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
342 writel(IMGU_IRQCTRL_IRQ_MASK,
343 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
344 /* Wait for write complete */
345 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
346
347 /* Enable IRQs from SP0 and SP1 controllers */
348 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
349 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
350 writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
351 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
352 writel(IMGU_IRQCTRL_IRQ_MASK,
353 base + IMGU_REG_IRQCTRL_ENABLE(i));
354 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
355 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
356 /* Wait for write complete */
357 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
358 }
359 }
360
imgu_css_hw_init(struct imgu_css * css)361 static int imgu_css_hw_init(struct imgu_css *css)
362 {
363 /* For checking that streaming monitor statuses are valid */
364 static const struct {
365 u32 reg;
366 u32 mask;
367 const char *name;
368 } stream_monitors[] = {
369 {
370 IMGU_REG_GP_SP1_STRMON_STAT,
371 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
372 "ISP0 to SP0"
373 }, {
374 IMGU_REG_GP_ISP_STRMON_STAT,
375 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
376 "SP0 to ISP0"
377 }, {
378 IMGU_REG_GP_MOD_STRMON_STAT,
379 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
380 "ISP0 to DMA0"
381 }, {
382 IMGU_REG_GP_ISP_STRMON_STAT,
383 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
384 "DMA0 to ISP0"
385 }, {
386 IMGU_REG_GP_MOD_STRMON_STAT,
387 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
388 "ISP0 to GDC0"
389 }, {
390 IMGU_REG_GP_MOD_STRMON_STAT,
391 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
392 "GDC0 to ISP0"
393 }, {
394 IMGU_REG_GP_MOD_STRMON_STAT,
395 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
396 "SP0 to DMA0"
397 }, {
398 IMGU_REG_GP_SP1_STRMON_STAT,
399 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
400 "DMA0 to SP0"
401 }, {
402 IMGU_REG_GP_MOD_STRMON_STAT,
403 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
404 "SP0 to GDC0"
405 }, {
406 IMGU_REG_GP_MOD_STRMON_STAT,
407 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
408 "GDC0 to SP0"
409 },
410 };
411
412 struct device *dev = css->dev;
413 void __iomem *const base = css->base;
414 u32 val, i;
415
416 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */
417 for (i = 0; i < IMGU_NUM_SP; i++) {
418 struct imgu_fw_info *bi =
419 &css->fwp->binary_header[css->fw_sp[i]];
420
421 writel(css->binary[css->fw_sp[i]].daddr,
422 base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
423 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
424 IMGU_CTRL_ICACHE_INV,
425 base + IMGU_REG_SP_CTRL(bi->type));
426 }
427 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
428 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
429 base + IMGU_REG_ISP_CTRL);
430
431 /* Check that IMGU hardware is ready */
432
433 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
434 dev_err(dev, "SP is not idle\n");
435 return -EIO;
436 }
437 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
438 dev_err(dev, "ISP is not idle\n");
439 return -EIO;
440 }
441
442 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
443 val = readl(base + stream_monitors[i].reg);
444 if (val & stream_monitors[i].mask) {
445 dev_err(dev, "error: Stream monitor %s is valid\n",
446 stream_monitors[i].name);
447 return -EIO;
448 }
449 }
450
451 /* Initialize GDC with default values */
452
453 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
454 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
455 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
456 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
457 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
458
459 writel(val0 | (val1 << 16),
460 base + IMGU_REG_GDC_LUT_BASE + i * 8);
461 writel(val2 | (val3 << 16),
462 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
463 }
464
465 return 0;
466 }
467
468 /* Boot the given IPU3 CSS SP */
imgu_css_hw_start_sp(struct imgu_css * css,int sp)469 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
470 {
471 void __iomem *const base = css->base;
472 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
473 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
474 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
475 + bi->blob.data_source,
476 .dmem_data_addr = bi->blob.data_target,
477 .dmem_bss_addr = bi->blob.bss_target,
478 .data_size = bi->blob.data_size,
479 .bss_size = bi->blob.bss_size,
480 .sp_id = sp,
481 };
482
483 writes(&dmem_cfg, sizeof(dmem_cfg), base +
484 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
485
486 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
487
488 writel(readl(base + IMGU_REG_SP_CTRL(sp))
489 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
490
491 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
492 + bi->info.sp.sw_state,
493 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
494 return -EIO;
495
496 return 0;
497 }
498
499 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
imgu_css_hw_start(struct imgu_css * css)500 static int imgu_css_hw_start(struct imgu_css *css)
501 {
502 static const u32 event_mask =
503 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
504 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
505 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
506 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
507 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
508 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
509 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
510 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
511 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
512 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
513 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
514 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
515
516 void __iomem *const base = css->base;
517 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
518 unsigned int i;
519
520 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
521
522 /* Start bootloader */
523
524 writel(IMGU_ABI_BL_SWSTATE_BUSY,
525 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
526 writel(IMGU_NUM_SP,
527 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
528
529 for (i = 0; i < IMGU_NUM_SP; i++) {
530 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */
531 struct imgu_fw_info *sp =
532 &css->fwp->binary_header[css->fw_sp[j]];
533 struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
534 .src_addr = css->binary[css->fw_sp[j]].daddr
535 + sp->blob.text_source,
536 .size = sp->blob.text_size,
537 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
538 .dst_addr = IMGU_SP_PMEM_BASE(j),
539 };
540
541 writes(&dma_cmd, sizeof(dma_cmd),
542 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
543 bl->info.bl.dma_cmd_list);
544 }
545
546 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
547
548 writel(readl(base + IMGU_REG_ISP_CTRL)
549 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
550 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
551 + bl->info.bl.sw_state, ~0,
552 IMGU_ABI_BL_SWSTATE_OK)) {
553 dev_err(css->dev, "failed to start bootloader\n");
554 return -EIO;
555 }
556
557 /* Start ISP */
558
559 memset(css->xmem_sp_group_ptrs.vaddr, 0,
560 sizeof(struct imgu_abi_sp_group));
561
562 bi = &css->fwp->binary_header[css->fw_sp[0]];
563
564 writel(css->xmem_sp_group_ptrs.daddr,
565 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
566
567 writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
568 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
569 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
570
571 if (imgu_css_hw_start_sp(css, 0))
572 return -EIO;
573
574 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
575 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
576 bi->info.sp.host_sp_queues_initialized);
577 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
578 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
579 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
580 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
581
582 /* Enable all events for all queues */
583
584 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
585 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
586 + bi->info.sp.host_sp_com
587 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
588 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
589 bi->info.sp.host_sp_queues_initialized);
590
591 /* Start SP1 */
592
593 bi = &css->fwp->binary_header[css->fw_sp[1]];
594
595 writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
596 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
597
598 if (imgu_css_hw_start_sp(css, 1))
599 return -EIO;
600
601 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
602 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
603
604 return 0;
605 }
606
imgu_css_hw_stop(struct imgu_css * css)607 static void imgu_css_hw_stop(struct imgu_css *css)
608 {
609 void __iomem *const base = css->base;
610 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
611
612 /* Stop fw */
613 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
614 base + IMGU_REG_SP_DMEM_BASE(0) +
615 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
616 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
617 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
618 dev_err(css->dev, "wait sp0 idle timeout.\n");
619 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
620 IMGU_ABI_SP_SWSTATE_TERMINATED)
621 dev_err(css->dev, "sp0 is not terminated.\n");
622 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
623 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
624 dev_err(css->dev, "wait isp idle timeout\n");
625 }
626
imgu_css_hw_cleanup(struct imgu_css * css)627 static void imgu_css_hw_cleanup(struct imgu_css *css)
628 {
629 void __iomem *const base = css->base;
630
631 /** Reset CSS **/
632
633 /* Clear the CSS busy signal */
634 readl(base + IMGU_REG_GP_BUSY);
635 writel(0, base + IMGU_REG_GP_BUSY);
636
637 /* Wait for idle signal */
638 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
639 IMGU_STATE_IDLE_STS))
640 dev_err(css->dev, "failed to shut down hw cleanly\n");
641
642 /* Reset the css */
643 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
644 base + IMGU_REG_PM_CTRL);
645
646 usleep_range(200, 300);
647 }
648
imgu_css_pipeline_cleanup(struct imgu_css * css,unsigned int pipe)649 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
650 {
651 struct imgu_device *imgu = dev_get_drvdata(css->dev);
652 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
653 unsigned int i;
654
655 imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
656 imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
657 imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
658 imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
659
660 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
661 imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
662 }
663
664 /*
665 * This function initializes various stages of the
666 * IPU3 CSS ISP pipeline
667 */
imgu_css_pipeline_init(struct imgu_css * css,unsigned int pipe)668 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
669 {
670 static const int BYPC = 2; /* Bytes per component */
671 static const struct imgu_abi_buffer_sp buffer_sp_init = {
672 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
673 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
674 };
675
676 struct imgu_abi_isp_iterator_config *cfg_iter;
677 struct imgu_abi_isp_ref_config *cfg_ref;
678 struct imgu_abi_isp_dvs_config *cfg_dvs;
679 struct imgu_abi_isp_tnr3_config *cfg_tnr;
680 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
681 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
682
683 const int stage = 0;
684 unsigned int i, j;
685
686 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
687 struct imgu_css_queue *css_queue_in =
688 &css_pipe->queue[IPU3_CSS_QUEUE_IN];
689 struct imgu_css_queue *css_queue_out =
690 &css_pipe->queue[IPU3_CSS_QUEUE_OUT];
691 struct imgu_css_queue *css_queue_vf =
692 &css_pipe->queue[IPU3_CSS_QUEUE_VF];
693 const struct imgu_fw_info *bi =
694 &css->fwp->binary_header[css_pipe->bindex];
695 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
696
697 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
698 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
699 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
700 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
701
702 struct imgu_abi_isp_stage *isp_stage;
703 struct imgu_abi_sp_stage *sp_stage;
704 struct imgu_abi_sp_group *sp_group;
705 struct imgu_abi_frames_sp *frames_sp;
706 struct imgu_abi_frame_sp *frame_sp;
707 struct imgu_abi_frame_sp_info *frame_sp_info;
708
709 const unsigned int bds_width_pad =
710 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
711 2 * IPU3_UAPI_ISP_VEC_ELEMS);
712
713 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
714 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
715 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
716
717 struct imgu_device *imgu = dev_get_drvdata(css->dev);
718
719 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
720
721 /* Configure iterator */
722
723 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
724 &cofs->dmem.iterator,
725 sizeof(*cfg_iter), vaddr);
726 if (!cfg_iter)
727 goto bad_firmware;
728
729 frame_sp_info = &cfg_iter->input_info;
730 frame_sp_info->res.width = css_queue_in->fmt.mpix.width;
731 frame_sp_info->res.height = css_queue_in->fmt.mpix.height;
732 frame_sp_info->padded_width = css_queue_in->width_pad;
733 frame_sp_info->format = css_queue_in->css_fmt->frame_format;
734 frame_sp_info->raw_bit_depth = css_queue_in->css_fmt->bit_depth;
735 frame_sp_info->raw_bayer_order = css_queue_in->css_fmt->bayer_order;
736 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
737
738 frame_sp_info = &cfg_iter->internal_info;
739 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
740 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
741 frame_sp_info->padded_width = bds_width_pad;
742 frame_sp_info->format = css_queue_out->css_fmt->frame_format;
743 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
744 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
745 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
746
747 frame_sp_info = &cfg_iter->output_info;
748 frame_sp_info->res.width = css_queue_out->fmt.mpix.width;
749 frame_sp_info->res.height = css_queue_out->fmt.mpix.height;
750 frame_sp_info->padded_width = css_queue_out->width_pad;
751 frame_sp_info->format = css_queue_out->css_fmt->frame_format;
752 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
753 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
754 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
755
756 frame_sp_info = &cfg_iter->vf_info;
757 frame_sp_info->res.width = css_queue_vf->fmt.mpix.width;
758 frame_sp_info->res.height = css_queue_vf->fmt.mpix.height;
759 frame_sp_info->padded_width = css_queue_vf->width_pad;
760 frame_sp_info->format = css_queue_vf->css_fmt->frame_format;
761 frame_sp_info->raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
762 frame_sp_info->raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
763 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
764
765 cfg_iter->dvs_envelope.width =
766 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
767 cfg_iter->dvs_envelope.height =
768 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
769
770 /* Configure reference (delay) frames */
771
772 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
773 &cofs->dmem.ref,
774 sizeof(*cfg_ref), vaddr);
775 if (!cfg_ref)
776 goto bad_firmware;
777
778 cfg_ref->port_b.crop = 0;
779 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
780 cfg_ref->port_b.width =
781 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
782 cfg_ref->port_b.stride =
783 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
784 cfg_ref->width_a_over_b =
785 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
786 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
787 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
788 cfg_ref->ref_frame_addr_y[i] =
789 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
790 cfg_ref->ref_frame_addr_c[i] =
791 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
792 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
793 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
794 }
795 for (; i < IMGU_ABI_FRAMES_REF; i++) {
796 cfg_ref->ref_frame_addr_y[i] = 0;
797 cfg_ref->ref_frame_addr_c[i] = 0;
798 }
799
800 /* Configure DVS (digital video stabilization) */
801
802 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
803 &cofs->dmem.dvs, sizeof(*cfg_dvs),
804 vaddr);
805 if (!cfg_dvs)
806 goto bad_firmware;
807
808 cfg_dvs->num_horizontal_blocks =
809 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
810 IMGU_DVS_BLOCK_W), 2);
811 cfg_dvs->num_vertical_blocks =
812 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
813 IMGU_DVS_BLOCK_H);
814
815 /* Configure TNR (temporal noise reduction) */
816
817 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
818 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
819 &cofs->dmem.tnr3,
820 sizeof(*cfg_tnr),
821 vaddr);
822 if (!cfg_tnr)
823 goto bad_firmware;
824
825 cfg_tnr->port_b.crop = 0;
826 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
827 cfg_tnr->port_b.width =
828 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
829 cfg_tnr->port_b.stride =
830 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
831 cfg_tnr->width_a_over_b =
832 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
833 cfg_tnr->frame_height =
834 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
835 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
836 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
837 cfg_tnr->frame_addr[i] =
838 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
839 .mem[i].daddr;
840 for (; i < IMGU_ABI_FRAMES_TNR; i++)
841 cfg_tnr->frame_addr[i] = 0;
842 }
843
844 /* Configure ref dmem state parameters */
845
846 cfg = IMGU_ABI_PARAM_CLASS_STATE;
847 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
848
849 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
850 &sofs->dmem.ref,
851 sizeof(*cfg_ref_state),
852 vaddr);
853 if (!cfg_ref_state)
854 goto bad_firmware;
855
856 cfg_ref_state->ref_in_buf_idx = 0;
857 cfg_ref_state->ref_out_buf_idx = 1;
858
859 /* Configure tnr dmem state parameters */
860 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
861 cfg_tnr_state =
862 imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
863 &sofs->dmem.tnr3,
864 sizeof(*cfg_tnr_state),
865 vaddr);
866 if (!cfg_tnr_state)
867 goto bad_firmware;
868
869 cfg_tnr_state->in_bufidx = 0;
870 cfg_tnr_state->out_bufidx = 1;
871 cfg_tnr_state->bypass_filter = 0;
872 cfg_tnr_state->total_frame_counter = 0;
873 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
874 cfg_tnr_state->buffer_frame_counter[i] = 0;
875 }
876
877 /* Configure ISP stage */
878
879 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
880 memset(isp_stage, 0, sizeof(*isp_stage));
881 isp_stage->blob_info = bi->blob;
882 isp_stage->binary_info = bi->info.isp.sp;
883 strscpy(isp_stage->binary_name,
884 (char *)css->fwp + bi->blob.prog_name_offset,
885 sizeof(isp_stage->binary_name));
886 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
887 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
888 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
889 isp_stage->mem_initializers.params[i][j].address =
890 css_pipe->binary_params_cs[i - 1][j].daddr;
891
892 /* Configure SP stage */
893
894 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
895 memset(sp_stage, 0, sizeof(*sp_stage));
896
897 frames_sp = &sp_stage->frames;
898 frames_sp->in.buf_attr = buffer_sp_init;
899 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
900 frames_sp->out[i].buf_attr = buffer_sp_init;
901 frames_sp->out_vf.buf_attr = buffer_sp_init;
902 frames_sp->s3a_buf = buffer_sp_init;
903 frames_sp->dvs_buf = buffer_sp_init;
904
905 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
906 sp_stage->num = stage;
907 sp_stage->isp_online = 0;
908 sp_stage->isp_copy_vf = 0;
909 sp_stage->isp_copy_output = 0;
910
911 sp_stage->enable.vf_output = css_pipe->vf_output_en;
912
913 frames_sp->effective_in_res.width =
914 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
915 frames_sp->effective_in_res.height =
916 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
917
918 frame_sp = &frames_sp->in;
919 frame_sp->info.res.width = css_queue_in->fmt.mpix.width;
920 frame_sp->info.res.height = css_queue_in->fmt.mpix.height;
921 frame_sp->info.padded_width = css_queue_in->width_pad;
922 frame_sp->info.format = css_queue_in->css_fmt->frame_format;
923 frame_sp->info.raw_bit_depth = css_queue_in->css_fmt->bit_depth;
924 frame_sp->info.raw_bayer_order = css_queue_in->css_fmt->bayer_order;
925 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
926 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
927 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
928
929 frame_sp = &frames_sp->out[0];
930 frame_sp->info.res.width = css_queue_out->fmt.mpix.width;
931 frame_sp->info.res.height = css_queue_out->fmt.mpix.height;
932 frame_sp->info.padded_width = css_queue_out->width_pad;
933 frame_sp->info.format = css_queue_out->css_fmt->frame_format;
934 frame_sp->info.raw_bit_depth = css_queue_out->css_fmt->bit_depth;
935 frame_sp->info.raw_bayer_order = css_queue_out->css_fmt->bayer_order;
936 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
937 frame_sp->planes.nv.uv.offset = css_queue_out->width_pad *
938 css_queue_out->fmt.mpix.height;
939 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
940 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
941
942 frame_sp = &frames_sp->out[1];
943 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
944
945 frame_sp_info = &frames_sp->internal_frame_info;
946 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
947 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
948 frame_sp_info->padded_width = bds_width_pad;
949 frame_sp_info->format = css_queue_out->css_fmt->frame_format;
950 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
951 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
952 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
953
954 frame_sp = &frames_sp->out_vf;
955 frame_sp->info.res.width = css_queue_vf->fmt.mpix.width;
956 frame_sp->info.res.height = css_queue_vf->fmt.mpix.height;
957 frame_sp->info.padded_width = css_queue_vf->width_pad;
958 frame_sp->info.format = css_queue_vf->css_fmt->frame_format;
959 frame_sp->info.raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
960 frame_sp->info.raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
961 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
962 frame_sp->planes.yuv.u.offset = css_queue_vf->width_pad *
963 css_queue_vf->fmt.mpix.height;
964 frame_sp->planes.yuv.v.offset = css_queue_vf->width_pad *
965 css_queue_vf->fmt.mpix.height * 5 / 4;
966 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
967 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
968
969 frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
970 frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
971
972 frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
973 frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
974
975 sp_stage->dvs_envelope.width =
976 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
977 sp_stage->dvs_envelope.height =
978 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
979
980 sp_stage->isp_pipe_version =
981 bi->info.isp.sp.pipeline.isp_pipe_version;
982 sp_stage->isp_deci_log_factor =
983 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
984 IMGU_MAX_BQ_GRID_WIDTH),
985 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
986 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
987 sp_stage->isp_vf_downscale_bits = 0;
988 sp_stage->if_config_index = 255;
989 sp_stage->sp_enable_xnr = 0;
990 sp_stage->num_stripes = stripes;
991 sp_stage->enable.s3a = 1;
992 sp_stage->enable.dvs_stats = 0;
993
994 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
995 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
996 sp_stage->isp_stage_addr =
997 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
998
999 /* Configure SP group */
1000
1001 sp_group = css->xmem_sp_group_ptrs.vaddr;
1002 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1003
1004 sp_group->pipe[pipe].num_stages = 1;
1005 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1006 sp_group->pipe[pipe].thread_id = pipe;
1007 sp_group->pipe[pipe].pipe_num = pipe;
1008 sp_group->pipe[pipe].num_execs = -1;
1009 sp_group->pipe[pipe].pipe_qos_config = -1;
1010 sp_group->pipe[pipe].required_bds_factor = 0;
1011 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1012 sp_group->pipe[pipe].inout_port_config =
1013 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1014 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1015 sp_group->pipe[pipe].scaler_pp_lut = 0;
1016 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1017 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1018 sp_group->pipe[pipe].sp_stage_addr[stage] =
1019 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1020 sp_group->pipe[pipe].pipe_config =
1021 bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1022 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1023
1024 /* Initialize parameter pools */
1025
1026 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1027 sizeof(struct imgu_abi_parameter_set_info)) ||
1028 imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1029 sizeof(struct imgu_abi_acc_param)) ||
1030 imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1031 sizeof(struct imgu_abi_gdc_warp_param) *
1032 3 * cfg_dvs->num_horizontal_blocks / 2 *
1033 cfg_dvs->num_vertical_blocks) ||
1034 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1035 imgu_css_fw_obgrid_size(
1036 &css->fwp->binary_header[css_pipe->bindex])))
1037 goto out_of_memory;
1038
1039 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1040 if (imgu_css_pool_init(imgu,
1041 &css_pipe->pool.binary_params_p[i],
1042 bi->info.isp.sp.mem_initializers.params
1043 [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1044 goto out_of_memory;
1045
1046 return 0;
1047
1048 bad_firmware:
1049 imgu_css_pipeline_cleanup(css, pipe);
1050 return -EPROTO;
1051
1052 out_of_memory:
1053 imgu_css_pipeline_cleanup(css, pipe);
1054 return -ENOMEM;
1055 }
1056
imgu_css_queue_pos(struct imgu_css * css,int queue,int thread)1057 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1058 {
1059 static const unsigned int sp;
1060 void __iomem *const base = css->base;
1061 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1062 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1063 bi->info.sp.host_sp_queue;
1064
1065 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1066 readb(&q->host2sp_evtq_info.end);
1067 }
1068
1069 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
imgu_css_queue_data(struct imgu_css * css,int queue,int thread,u32 data)1070 static int imgu_css_queue_data(struct imgu_css *css,
1071 int queue, int thread, u32 data)
1072 {
1073 static const unsigned int sp;
1074 void __iomem *const base = css->base;
1075 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1076 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1077 bi->info.sp.host_sp_queue;
1078 u8 size, start, end, end2;
1079
1080 if (queue >= 0) {
1081 size = readb(&q->host2sp_bufq_info[thread][queue].size);
1082 start = readb(&q->host2sp_bufq_info[thread][queue].start);
1083 end = readb(&q->host2sp_bufq_info[thread][queue].end);
1084 } else {
1085 size = readb(&q->host2sp_evtq_info.size);
1086 start = readb(&q->host2sp_evtq_info.start);
1087 end = readb(&q->host2sp_evtq_info.end);
1088 }
1089
1090 if (size == 0)
1091 return -EIO;
1092
1093 end2 = (end + 1) % size;
1094 if (end2 == start)
1095 return -EBUSY; /* Queue full */
1096
1097 if (queue >= 0) {
1098 writel(data, &q->host2sp_bufq[thread][queue][end]);
1099 writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1100 } else {
1101 writel(data, &q->host2sp_evtq[end]);
1102 writeb(end2, &q->host2sp_evtq_info.end);
1103 }
1104
1105 return 0;
1106 }
1107
1108 /* Receive data using given buffer queue, or if queue < 0, event queue. */
imgu_css_dequeue_data(struct imgu_css * css,int queue,u32 * data)1109 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1110 {
1111 static const unsigned int sp;
1112 void __iomem *const base = css->base;
1113 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1114 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1115 bi->info.sp.host_sp_queue;
1116 u8 size, start, end, start2;
1117
1118 if (queue >= 0) {
1119 size = readb(&q->sp2host_bufq_info[queue].size);
1120 start = readb(&q->sp2host_bufq_info[queue].start);
1121 end = readb(&q->sp2host_bufq_info[queue].end);
1122 } else {
1123 size = readb(&q->sp2host_evtq_info.size);
1124 start = readb(&q->sp2host_evtq_info.start);
1125 end = readb(&q->sp2host_evtq_info.end);
1126 }
1127
1128 if (size == 0)
1129 return -EIO;
1130
1131 if (end == start)
1132 return -EBUSY; /* Queue empty */
1133
1134 start2 = (start + 1) % size;
1135
1136 if (queue >= 0) {
1137 *data = readl(&q->sp2host_bufq[queue][start]);
1138 writeb(start2, &q->sp2host_bufq_info[queue].start);
1139 } else {
1140 int r;
1141
1142 *data = readl(&q->sp2host_evtq[start]);
1143 writeb(start2, &q->sp2host_evtq_info.start);
1144
1145 /* Acknowledge events dequeued from event queue */
1146 r = imgu_css_queue_data(css, queue, 0,
1147 IMGU_ABI_EVENT_EVENT_DEQUEUED);
1148 if (r < 0)
1149 return r;
1150 }
1151
1152 return 0;
1153 }
1154
1155 /* Free binary-specific resources */
imgu_css_binary_cleanup(struct imgu_css * css,unsigned int pipe)1156 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1157 {
1158 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1159 unsigned int i, j;
1160
1161 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1162
1163 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1164 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1165 imgu_dmamap_free(imgu,
1166 &css_pipe->binary_params_cs[j][i]);
1167
1168 j = IPU3_CSS_AUX_FRAME_REF;
1169 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1170 imgu_dmamap_free(imgu,
1171 &css_pipe->aux_frames[j].mem[i]);
1172
1173 j = IPU3_CSS_AUX_FRAME_TNR;
1174 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1175 imgu_dmamap_free(imgu,
1176 &css_pipe->aux_frames[j].mem[i]);
1177 }
1178
imgu_css_binary_preallocate(struct imgu_css * css,unsigned int pipe)1179 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1180 {
1181 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1182 unsigned int i, j;
1183
1184 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1185
1186 for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1187 j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1188 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1189 if (!imgu_dmamap_alloc(imgu,
1190 &css_pipe->binary_params_cs[j - 1][i],
1191 CSS_ABI_SIZE))
1192 goto out_of_memory;
1193
1194 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1195 if (!imgu_dmamap_alloc(imgu,
1196 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].
1197 mem[i], CSS_BDS_SIZE))
1198 goto out_of_memory;
1199
1200 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1201 if (!imgu_dmamap_alloc(imgu,
1202 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].
1203 mem[i], CSS_GDC_SIZE))
1204 goto out_of_memory;
1205
1206 return 0;
1207
1208 out_of_memory:
1209 imgu_css_binary_cleanup(css, pipe);
1210 return -ENOMEM;
1211 }
1212
1213 /* allocate binary-specific resources */
imgu_css_binary_setup(struct imgu_css * css,unsigned int pipe)1214 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1215 {
1216 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1217 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1218 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1219 int i, j, size;
1220 static const int BYPC = 2; /* Bytes per component */
1221 unsigned int w, h;
1222
1223 /* Allocate parameter memory blocks for this binary */
1224
1225 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1226 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1227 if (imgu_css_dma_buffer_resize(
1228 imgu,
1229 &css_pipe->binary_params_cs[j - 1][i],
1230 bi->info.isp.sp.mem_initializers.params[j][i].size))
1231 goto out_of_memory;
1232 }
1233
1234 /* Allocate internal frame buffers */
1235
1236 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1237 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1238 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1239 css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1240 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1241 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1242 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1243 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1244 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1245 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1246 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1247 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1248 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1249 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1250 if (imgu_css_dma_buffer_resize(
1251 imgu,
1252 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1253 size))
1254 goto out_of_memory;
1255
1256 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1257 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1258 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1259 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1260 bi->info.isp.sp.block.block_width *
1261 IPU3_UAPI_ISP_VEC_ELEMS);
1262 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1263 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1264 bi->info.isp.sp.block.output_block_height);
1265
1266 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1267 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1268 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1269 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */
1270 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1271 if (imgu_css_dma_buffer_resize(
1272 imgu,
1273 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1274 size))
1275 goto out_of_memory;
1276
1277 return 0;
1278
1279 out_of_memory:
1280 imgu_css_binary_cleanup(css, pipe);
1281 return -ENOMEM;
1282 }
1283
imgu_css_start_streaming(struct imgu_css * css)1284 int imgu_css_start_streaming(struct imgu_css *css)
1285 {
1286 u32 data;
1287 int r, pipe;
1288
1289 if (css->streaming)
1290 return -EPROTO;
1291
1292 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1293 r = imgu_css_binary_setup(css, pipe);
1294 if (r < 0)
1295 return r;
1296 }
1297
1298 r = imgu_css_hw_init(css);
1299 if (r < 0)
1300 return r;
1301
1302 r = imgu_css_hw_start(css);
1303 if (r < 0)
1304 goto fail;
1305
1306 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1307 r = imgu_css_pipeline_init(css, pipe);
1308 if (r < 0)
1309 goto fail;
1310 }
1311
1312 css->streaming = true;
1313
1314 imgu_css_hw_enable_irq(css);
1315
1316 /* Initialize parameters to default */
1317 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1318 r = imgu_css_set_parameters(css, pipe, NULL);
1319 if (r < 0)
1320 goto fail;
1321 }
1322
1323 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1324 ;
1325 if (r != -EBUSY)
1326 goto fail;
1327
1328 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1329 ;
1330 if (r != -EBUSY)
1331 goto fail;
1332
1333 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1334 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1335 IMGU_ABI_EVENT_START_STREAM |
1336 pipe << 16);
1337 if (r < 0)
1338 goto fail;
1339 }
1340
1341 return 0;
1342
1343 fail:
1344 css->streaming = false;
1345 imgu_css_hw_cleanup(css);
1346 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1347 imgu_css_pipeline_cleanup(css, pipe);
1348 imgu_css_binary_cleanup(css, pipe);
1349 }
1350
1351 return r;
1352 }
1353
imgu_css_stop_streaming(struct imgu_css * css)1354 void imgu_css_stop_streaming(struct imgu_css *css)
1355 {
1356 struct imgu_css_buffer *b, *b0;
1357 int q, r, pipe;
1358
1359 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1360 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1361 IMGU_ABI_EVENT_STOP_STREAM);
1362 if (r < 0)
1363 dev_warn(css->dev, "failed on stop stream event\n");
1364 }
1365
1366 if (!css->streaming)
1367 return;
1368
1369 imgu_css_hw_stop(css);
1370
1371 imgu_css_hw_cleanup(css);
1372
1373 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1374 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1375
1376 imgu_css_pipeline_cleanup(css, pipe);
1377
1378 spin_lock(&css_pipe->qlock);
1379 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1380 list_for_each_entry_safe(b, b0,
1381 &css_pipe->queue[q].bufs,
1382 list) {
1383 b->state = IPU3_CSS_BUFFER_FAILED;
1384 list_del(&b->list);
1385 }
1386 spin_unlock(&css_pipe->qlock);
1387 }
1388
1389 css->streaming = false;
1390 }
1391
imgu_css_pipe_queue_empty(struct imgu_css * css,unsigned int pipe)1392 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1393 {
1394 int q;
1395 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1396
1397 spin_lock(&css_pipe->qlock);
1398 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1399 if (!list_empty(&css_pipe->queue[q].bufs))
1400 break;
1401 spin_unlock(&css_pipe->qlock);
1402 return (q == IPU3_CSS_QUEUES);
1403 }
1404
imgu_css_queue_empty(struct imgu_css * css)1405 bool imgu_css_queue_empty(struct imgu_css *css)
1406 {
1407 unsigned int pipe;
1408 bool ret = false;
1409
1410 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1411 ret &= imgu_css_pipe_queue_empty(css, pipe);
1412
1413 return ret;
1414 }
1415
imgu_css_is_streaming(struct imgu_css * css)1416 bool imgu_css_is_streaming(struct imgu_css *css)
1417 {
1418 return css->streaming;
1419 }
1420
imgu_css_map_init(struct imgu_css * css,unsigned int pipe)1421 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1422 {
1423 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1424 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1425 unsigned int p, q, i;
1426
1427 /* Allocate and map common structures with imgu hardware */
1428 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1429 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1430 if (!imgu_dmamap_alloc(imgu,
1431 &css_pipe->
1432 xmem_sp_stage_ptrs[p][i],
1433 sizeof(struct imgu_abi_sp_stage)))
1434 return -ENOMEM;
1435 if (!imgu_dmamap_alloc(imgu,
1436 &css_pipe->
1437 xmem_isp_stage_ptrs[p][i],
1438 sizeof(struct imgu_abi_isp_stage)))
1439 return -ENOMEM;
1440 }
1441
1442 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1443 ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1444 IMGU_ABI_ISP_DDR_WORD_BYTES)))
1445 return -ENOMEM;
1446
1447 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1448 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1449
1450 for (i = 0; i < abi_buf_num; i++)
1451 if (!imgu_dmamap_alloc(imgu,
1452 &css_pipe->abi_buffers[q][i],
1453 sizeof(struct imgu_abi_buffer)))
1454 return -ENOMEM;
1455 }
1456
1457 if (imgu_css_binary_preallocate(css, pipe)) {
1458 imgu_css_binary_cleanup(css, pipe);
1459 return -ENOMEM;
1460 }
1461
1462 return 0;
1463 }
1464
imgu_css_pipe_cleanup(struct imgu_css * css,unsigned int pipe)1465 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1466 {
1467 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1468 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1469 unsigned int p, q, i, abi_buf_num;
1470
1471 imgu_css_binary_cleanup(css, pipe);
1472
1473 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1474 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1475 for (i = 0; i < abi_buf_num; i++)
1476 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1477 }
1478
1479 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1480 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1481 imgu_dmamap_free(imgu,
1482 &css_pipe->xmem_sp_stage_ptrs[p][i]);
1483 imgu_dmamap_free(imgu,
1484 &css_pipe->xmem_isp_stage_ptrs[p][i]);
1485 }
1486
1487 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1488 }
1489
imgu_css_cleanup(struct imgu_css * css)1490 void imgu_css_cleanup(struct imgu_css *css)
1491 {
1492 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1493 unsigned int pipe;
1494
1495 imgu_css_stop_streaming(css);
1496 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1497 imgu_css_pipe_cleanup(css, pipe);
1498 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1499 imgu_css_fw_cleanup(css);
1500 }
1501
imgu_css_init(struct device * dev,struct imgu_css * css,void __iomem * base,int length)1502 int imgu_css_init(struct device *dev, struct imgu_css *css,
1503 void __iomem *base, int length)
1504 {
1505 struct imgu_device *imgu = dev_get_drvdata(dev);
1506 int r, q, pipe;
1507
1508 /* Initialize main data structure */
1509 css->dev = dev;
1510 css->base = base;
1511 css->iomem_length = length;
1512
1513 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1514 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1515
1516 css_pipe->vf_output_en = false;
1517 spin_lock_init(&css_pipe->qlock);
1518 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1519 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1520 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1521 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1522 if (r)
1523 return r;
1524 }
1525 r = imgu_css_map_init(css, pipe);
1526 if (r) {
1527 imgu_css_cleanup(css);
1528 return r;
1529 }
1530 }
1531 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1532 sizeof(struct imgu_abi_sp_group)))
1533 return -ENOMEM;
1534
1535 r = imgu_css_fw_init(css);
1536 if (r)
1537 return r;
1538
1539 return 0;
1540 }
1541
imgu_css_adjust(u32 res,u32 align)1542 static u32 imgu_css_adjust(u32 res, u32 align)
1543 {
1544 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1545
1546 return DIV_ROUND_CLOSEST(val, align) * align;
1547 }
1548
1549 /* Select a binary matching the required resolutions and formats */
imgu_css_find_binary(struct imgu_css * css,unsigned int pipe,struct imgu_css_queue queue[IPU3_CSS_QUEUES],struct v4l2_rect rects[IPU3_CSS_RECTS])1550 static int imgu_css_find_binary(struct imgu_css *css,
1551 unsigned int pipe,
1552 struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1553 struct v4l2_rect rects[IPU3_CSS_RECTS])
1554 {
1555 const int binary_nr = css->fwp->file_header.binary_nr;
1556 unsigned int binary_mode =
1557 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1558 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1559 const struct v4l2_pix_format_mplane *in =
1560 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1561 const struct v4l2_pix_format_mplane *out =
1562 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1563 const struct v4l2_pix_format_mplane *vf =
1564 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1565 u32 stripe_w = 0, stripe_h = 0;
1566 const char *name;
1567 int i, j;
1568
1569 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1570 return -EINVAL;
1571
1572 /* Find out the strip size boundary */
1573 for (i = 0; i < binary_nr; i++) {
1574 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1575
1576 u32 max_width = bi->info.isp.sp.output.max_width;
1577 u32 max_height = bi->info.isp.sp.output.max_height;
1578
1579 if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1580 stripe_w = stripe_w ?
1581 min(stripe_w, max_width) : max_width;
1582 stripe_h = stripe_h ?
1583 min(stripe_h, max_height) : max_height;
1584 }
1585 }
1586
1587 for (i = 0; i < binary_nr; i++) {
1588 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1589 enum imgu_abi_frame_format q_fmt;
1590
1591 name = (void *)css->fwp + bi->blob.prog_name_offset;
1592
1593 /* Check that binary supports memory-to-memory processing */
1594 if (bi->info.isp.sp.input.source !=
1595 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1596 continue;
1597
1598 /* Check that binary supports raw10 input */
1599 if (!bi->info.isp.sp.enable.input_feeder &&
1600 !bi->info.isp.sp.enable.input_raw)
1601 continue;
1602
1603 /* Check binary mode */
1604 if (bi->info.isp.sp.pipeline.mode != binary_mode)
1605 continue;
1606
1607 /* Since input is RGGB bayer, need to process colors */
1608 if (bi->info.isp.sp.enable.luma_only)
1609 continue;
1610
1611 if (in->width < bi->info.isp.sp.input.min_width ||
1612 in->width > bi->info.isp.sp.input.max_width ||
1613 in->height < bi->info.isp.sp.input.min_height ||
1614 in->height > bi->info.isp.sp.input.max_height)
1615 continue;
1616
1617 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1618 if (bi->info.isp.num_output_pins <= 0)
1619 continue;
1620
1621 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1622 for (j = 0; j < bi->info.isp.num_output_formats; j++)
1623 if (bi->info.isp.output_formats[j] == q_fmt)
1624 break;
1625 if (j >= bi->info.isp.num_output_formats)
1626 continue;
1627
1628 if (out->width < bi->info.isp.sp.output.min_width ||
1629 out->width > bi->info.isp.sp.output.max_width ||
1630 out->height < bi->info.isp.sp.output.min_height ||
1631 out->height > bi->info.isp.sp.output.max_height)
1632 continue;
1633
1634 if (out->width > bi->info.isp.sp.internal.max_width ||
1635 out->height > bi->info.isp.sp.internal.max_height)
1636 continue;
1637 }
1638
1639 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1640 if (bi->info.isp.num_output_pins <= 1)
1641 continue;
1642
1643 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1644 for (j = 0; j < bi->info.isp.num_output_formats; j++)
1645 if (bi->info.isp.output_formats[j] == q_fmt)
1646 break;
1647 if (j >= bi->info.isp.num_output_formats)
1648 continue;
1649
1650 if (vf->width < bi->info.isp.sp.output.min_width ||
1651 vf->width > bi->info.isp.sp.output.max_width ||
1652 vf->height < bi->info.isp.sp.output.min_height ||
1653 vf->height > bi->info.isp.sp.output.max_height)
1654 continue;
1655 }
1656
1657 /* All checks passed, select the binary */
1658 dev_dbg(css->dev, "using binary %s id = %u\n", name,
1659 bi->info.isp.sp.id);
1660 return i;
1661 }
1662
1663 /* Can not find suitable binary for these parameters */
1664 return -EINVAL;
1665 }
1666
1667 /*
1668 * Check that there is a binary matching requirements. Parameters may be
1669 * NULL indicating disabled input/output. Return negative if given
1670 * parameters can not be supported or on error, zero or positive indicating
1671 * found binary number. May modify the given parameters if not exact match
1672 * is found.
1673 */
imgu_css_fmt_try(struct imgu_css * css,struct v4l2_pix_format_mplane * fmts[IPU3_CSS_QUEUES],struct v4l2_rect * rects[IPU3_CSS_RECTS],unsigned int pipe)1674 int imgu_css_fmt_try(struct imgu_css *css,
1675 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1676 struct v4l2_rect *rects[IPU3_CSS_RECTS],
1677 unsigned int pipe)
1678 {
1679 static const u32 EFF_ALIGN_W = 2;
1680 static const u32 BDS_ALIGN_W = 4;
1681 static const u32 OUT_ALIGN_W = 8;
1682 static const u32 OUT_ALIGN_H = 4;
1683 static const u32 VF_ALIGN_W = 2;
1684 static const char *qnames[IPU3_CSS_QUEUES] = {
1685 [IPU3_CSS_QUEUE_IN] = "in",
1686 [IPU3_CSS_QUEUE_PARAMS] = "params",
1687 [IPU3_CSS_QUEUE_OUT] = "out",
1688 [IPU3_CSS_QUEUE_VF] = "vf",
1689 [IPU3_CSS_QUEUE_STAT_3A] = "3a",
1690 };
1691 static const char *rnames[IPU3_CSS_RECTS] = {
1692 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1693 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution",
1694 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size",
1695 [IPU3_CSS_RECT_GDC] = "GDC output res",
1696 };
1697 struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1698 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1699 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1700 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1701 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1702 struct imgu_css_queue *q;
1703 struct v4l2_pix_format_mplane *in, *out, *vf;
1704 int i, s, ret;
1705
1706 q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
1707 if (!q)
1708 return -ENOMEM;
1709
1710 in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1711 out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1712 vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1713
1714 /* Adjust all formats, get statistics buffer sizes and formats */
1715 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1716 if (fmts[i])
1717 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1718 qnames[i], fmts[i]->width, fmts[i]->height,
1719 fmts[i]->pixelformat);
1720 else
1721 dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1722 qnames[i]);
1723 if (imgu_css_queue_init(&q[i], fmts[i],
1724 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1725 dev_notice(css->dev, "can not initialize queue %s\n",
1726 qnames[i]);
1727 ret = -EINVAL;
1728 goto out;
1729 }
1730 }
1731 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1732 if (rects[i]) {
1733 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1734 rnames[i], rects[i]->width, rects[i]->height);
1735 r[i].width = rects[i]->width;
1736 r[i].height = rects[i]->height;
1737 } else {
1738 dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1739 rnames[i]);
1740 }
1741 /* For now, force known good resolutions */
1742 r[i].left = 0;
1743 r[i].top = 0;
1744 }
1745
1746 /* Always require one input and vf only if out is also enabled */
1747 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1748 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1749 dev_warn(css->dev, "required queues are disabled\n");
1750 ret = -EINVAL;
1751 goto out;
1752 }
1753
1754 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1755 out->width = in->width;
1756 out->height = in->height;
1757 }
1758 if (eff->width <= 0 || eff->height <= 0) {
1759 eff->width = in->width;
1760 eff->height = in->height;
1761 }
1762 if (bds->width <= 0 || bds->height <= 0) {
1763 bds->width = out->width;
1764 bds->height = out->height;
1765 }
1766 if (gdc->width <= 0 || gdc->height <= 0) {
1767 gdc->width = out->width;
1768 gdc->height = out->height;
1769 }
1770
1771 in->width = imgu_css_adjust(in->width, 1);
1772 in->height = imgu_css_adjust(in->height, 1);
1773 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1774 eff->height = imgu_css_adjust(eff->height, 1);
1775 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1776 bds->height = imgu_css_adjust(bds->height, 1);
1777 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1778 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1779 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W);
1780 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1781 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W);
1782 vf->height = imgu_css_adjust(vf->height, 1);
1783
1784 s = (bds->width - gdc->width) / 2;
1785 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1786 s = (bds->height - gdc->height) / 2;
1787 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1788
1789 ret = imgu_css_find_binary(css, pipe, q, r);
1790 if (ret < 0) {
1791 dev_err(css->dev, "failed to find suitable binary\n");
1792 ret = -EINVAL;
1793 goto out;
1794 }
1795 css->pipes[pipe].bindex = ret;
1796
1797 dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1798 css->pipes[pipe].bindex, pipe);
1799
1800 /* Final adjustment and set back the queried formats */
1801 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1802 if (fmts[i]) {
1803 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1804 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1805 dev_err(css->dev,
1806 "final resolution adjustment failed\n");
1807 ret = -EINVAL;
1808 goto out;
1809 }
1810 *fmts[i] = q[i].fmt.mpix;
1811 }
1812 }
1813
1814 for (i = 0; i < IPU3_CSS_RECTS; i++)
1815 if (rects[i])
1816 *rects[i] = r[i];
1817
1818 dev_dbg(css->dev,
1819 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1820 in->width, in->height, eff->width, eff->height,
1821 bds->width, bds->height, gdc->width, gdc->height,
1822 out->width, out->height, vf->width, vf->height);
1823
1824 ret = 0;
1825 out:
1826 kfree(q);
1827 return ret;
1828 }
1829
imgu_css_fmt_set(struct imgu_css * css,struct v4l2_pix_format_mplane * fmts[IPU3_CSS_QUEUES],struct v4l2_rect * rects[IPU3_CSS_RECTS],unsigned int pipe)1830 int imgu_css_fmt_set(struct imgu_css *css,
1831 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1832 struct v4l2_rect *rects[IPU3_CSS_RECTS],
1833 unsigned int pipe)
1834 {
1835 struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1836 struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1837 int i, r;
1838 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1839
1840 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1841 if (rects[i])
1842 rect_data[i] = *rects[i];
1843 else
1844 memset(&rect_data[i], 0, sizeof(rect_data[i]));
1845 all_rects[i] = &rect_data[i];
1846 }
1847 r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1848 if (r < 0)
1849 return r;
1850
1851 for (i = 0; i < IPU3_CSS_QUEUES; i++)
1852 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1853 IPU3_CSS_QUEUE_TO_FLAGS(i)))
1854 return -EINVAL;
1855 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1856 css_pipe->rect[i] = rect_data[i];
1857 if (rects[i])
1858 *rects[i] = rect_data[i];
1859 }
1860
1861 return 0;
1862 }
1863
imgu_css_meta_fmt_set(struct v4l2_meta_format * fmt)1864 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1865 {
1866 switch (fmt->dataformat) {
1867 case V4L2_META_FMT_IPU3_PARAMS:
1868 fmt->buffersize = sizeof(struct ipu3_uapi_params);
1869
1870 /*
1871 * Sanity check for the parameter struct size. This must
1872 * not change!
1873 */
1874 BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1875
1876 break;
1877 case V4L2_META_FMT_IPU3_STAT_3A:
1878 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1879 break;
1880 default:
1881 return -EINVAL;
1882 }
1883
1884 return 0;
1885 }
1886
1887 /*
1888 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1889 * called for the buffer. May be called from interrupt context.
1890 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1891 * code on error conditions.
1892 */
imgu_css_buf_queue(struct imgu_css * css,unsigned int pipe,struct imgu_css_buffer * b)1893 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1894 struct imgu_css_buffer *b)
1895 {
1896 struct imgu_abi_buffer *abi_buf;
1897 struct imgu_addr_t *buf_addr;
1898 u32 data;
1899 int r;
1900 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1901
1902 if (!css->streaming)
1903 return -EPROTO; /* CSS or buffer in wrong state */
1904
1905 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1906 return -EINVAL;
1907
1908 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1909 pipe);
1910
1911 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1912 return -EIO;
1913 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1914
1915 /* Fill struct abi_buffer for firmware */
1916 memset(abi_buf, 0, sizeof(*abi_buf));
1917
1918 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1919 *(imgu_addr_t *)buf_addr = b->daddr;
1920
1921 if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1922 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1923
1924 if (b->queue == IPU3_CSS_QUEUE_OUT)
1925 abi_buf->payload.frame.padded_width =
1926 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1927
1928 if (b->queue == IPU3_CSS_QUEUE_VF)
1929 abi_buf->payload.frame.padded_width =
1930 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1931
1932 spin_lock(&css_pipe->qlock);
1933 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1934 spin_unlock(&css_pipe->qlock);
1935 b->state = IPU3_CSS_BUFFER_QUEUED;
1936
1937 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1938 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1939 pipe, data);
1940 if (r < 0)
1941 goto queueing_failed;
1942
1943 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1944 imgu_css_queues[b->queue].qid);
1945 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1946 if (r < 0)
1947 goto queueing_failed;
1948
1949 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1950 b, b->queue, pipe);
1951
1952 return 0;
1953
1954 queueing_failed:
1955 b->state = (r == -EBUSY || r == -EAGAIN) ?
1956 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1957 list_del(&b->list);
1958
1959 return r;
1960 }
1961
1962 /*
1963 * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1964 * should be called again, or -EBUSY which means that there are no more
1965 * buffers available. May be called from interrupt context.
1966 */
imgu_css_buf_dequeue(struct imgu_css * css)1967 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1968 {
1969 static const unsigned char evtype_to_queue[] = {
1970 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1971 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1972 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1973 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1974 };
1975 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
1976 u32 event, daddr;
1977 int evtype, pipe, pipeid, queue, qid, r;
1978 struct imgu_css_pipe *css_pipe;
1979
1980 if (!css->streaming)
1981 return ERR_PTR(-EPROTO);
1982
1983 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
1984 if (r < 0)
1985 return ERR_PTR(r);
1986
1987 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1988 IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1989
1990 switch (evtype) {
1991 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1992 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1993 case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1994 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1995 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1996 IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1997 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1998 IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1999 queue = evtype_to_queue[evtype];
2000 qid = imgu_css_queues[queue].qid;
2001
2002 if (pipe >= IMGU_MAX_PIPE_NUM) {
2003 dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2004 return ERR_PTR(-EIO);
2005 }
2006
2007 if (qid >= IMGU_ABI_QUEUE_NUM) {
2008 dev_err(css->dev, "Invalid qid: %i\n", qid);
2009 return ERR_PTR(-EIO);
2010 }
2011 css_pipe = &css->pipes[pipe];
2012 dev_dbg(css->dev,
2013 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2014 event, queue, pipe, pipeid);
2015
2016 r = imgu_css_dequeue_data(css, qid, &daddr);
2017 if (r < 0) {
2018 dev_err(css->dev, "failed to dequeue buffer\n");
2019 /* Force real error, not -EBUSY */
2020 return ERR_PTR(-EIO);
2021 }
2022
2023 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2024 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2025 if (r < 0) {
2026 dev_err(css->dev, "failed to queue event\n");
2027 return ERR_PTR(-EIO);
2028 }
2029
2030 spin_lock(&css_pipe->qlock);
2031 if (list_empty(&css_pipe->queue[queue].bufs)) {
2032 spin_unlock(&css_pipe->qlock);
2033 dev_err(css->dev, "event on empty queue\n");
2034 return ERR_PTR(-EIO);
2035 }
2036 b = list_first_entry(&css_pipe->queue[queue].bufs,
2037 struct imgu_css_buffer, list);
2038 if (queue != b->queue ||
2039 daddr != css_pipe->abi_buffers
2040 [b->queue][b->queue_pos].daddr) {
2041 spin_unlock(&css_pipe->qlock);
2042 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2043 return ERR_PTR(-EIO);
2044 }
2045
2046 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2047 b->pipe = pipe;
2048 b->state = IPU3_CSS_BUFFER_DONE;
2049 list_del(&b->list);
2050 spin_unlock(&css_pipe->qlock);
2051 break;
2052 case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2053 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2054 IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2055 if (pipe >= IMGU_MAX_PIPE_NUM) {
2056 dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2057 return ERR_PTR(-EIO);
2058 }
2059
2060 css_pipe = &css->pipes[pipe];
2061 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2062 event, pipe);
2063 break;
2064 case IMGU_ABI_EVTTYPE_TIMER:
2065 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2066 if (r < 0)
2067 return ERR_PTR(r);
2068
2069 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2070 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2071 dev_dbg(css->dev, "event: timer\n");
2072 else
2073 dev_warn(css->dev, "half of timer event missing\n");
2074 break;
2075 case IMGU_ABI_EVTTYPE_FW_WARNING:
2076 dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2077 break;
2078 case IMGU_ABI_EVTTYPE_FW_ASSERT:
2079 dev_err(css->dev,
2080 "event: firmware assert 0x%x module_id %i line_no %i\n",
2081 event,
2082 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2083 IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2084 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2085 IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2086 break;
2087 default:
2088 dev_warn(css->dev, "received unknown event 0x%x\n", event);
2089 }
2090
2091 return b;
2092 }
2093
2094 /*
2095 * Get a new set of parameters from pool and initialize them based on
2096 * the parameters params, gdc, and obgrid. Any of these may be NULL,
2097 * in which case the previously set parameters are used.
2098 * If parameters haven't been set previously, initialize from scratch.
2099 *
2100 * Return index to css->parameter_set_info which has the newly created
2101 * parameters or negative value on error.
2102 */
imgu_css_set_parameters(struct imgu_css * css,unsigned int pipe,struct ipu3_uapi_params * set_params)2103 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2104 struct ipu3_uapi_params *set_params)
2105 {
2106 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2107 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2108 const int stage = 0;
2109 const struct imgu_fw_info *bi;
2110 int obgrid_size;
2111 unsigned int stripes, i;
2112 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2113
2114 /* Destination buffers which are filled here */
2115 struct imgu_abi_parameter_set_info *param_set;
2116 struct imgu_abi_acc_param *acc = NULL;
2117 struct imgu_abi_gdc_warp_param *gdc = NULL;
2118 struct ipu3_uapi_obgrid_param *obgrid = NULL;
2119 const struct imgu_css_map *map;
2120 void *vmem0 = NULL;
2121 void *dmem0 = NULL;
2122
2123 enum imgu_abi_memories m;
2124 int r = -EBUSY;
2125
2126 if (!css->streaming)
2127 return -EPROTO;
2128
2129 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2130
2131 bi = &css->fwp->binary_header[css_pipe->bindex];
2132 obgrid_size = imgu_css_fw_obgrid_size(bi);
2133 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2134
2135 imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2136 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2137 0)->vaddr;
2138
2139 /* Get a new acc only if new parameters given, or none yet */
2140 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2141 if (set_params || !map->vaddr) {
2142 imgu_css_pool_get(&css_pipe->pool.acc);
2143 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2144 acc = map->vaddr;
2145 }
2146
2147 /* Get new VMEM0 only if needed, or none yet */
2148 m = IMGU_ABI_MEM_ISP_VMEM0;
2149 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2150 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2151 set_params->use.tnr3_vmem_params ||
2152 set_params->use.xnr3_vmem_params))) {
2153 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2154 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2155 vmem0 = map->vaddr;
2156 }
2157
2158 /* Get new DMEM0 only if needed, or none yet */
2159 m = IMGU_ABI_MEM_ISP_DMEM0;
2160 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2161 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2162 set_params->use.xnr3_dmem_params))) {
2163 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2164 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2165 dmem0 = map->vaddr;
2166 }
2167
2168 /* Configure acc parameter cluster */
2169 if (acc) {
2170 /* get acc_old */
2171 map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2172 /* user acc */
2173 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2174 set_params ? &set_params->acc_param : NULL);
2175 if (r < 0)
2176 goto fail;
2177 }
2178
2179 /* Configure late binding parameters */
2180 if (vmem0) {
2181 m = IMGU_ABI_MEM_ISP_VMEM0;
2182 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2183 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2184 map->vaddr, set_params);
2185 if (r < 0)
2186 goto fail;
2187 }
2188
2189 if (dmem0) {
2190 m = IMGU_ABI_MEM_ISP_DMEM0;
2191 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2192 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2193 map->vaddr, set_params);
2194 if (r < 0)
2195 goto fail;
2196 }
2197
2198 /* Get a new gdc only if a new gdc is given, or none yet */
2199 if (bi->info.isp.sp.enable.dvs_6axis) {
2200 unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2201 unsigned int g = IPU3_CSS_RECT_GDC;
2202 unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2203
2204 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2205 if (!map->vaddr) {
2206 imgu_css_pool_get(&css_pipe->pool.gdc);
2207 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2208 gdc = map->vaddr;
2209 imgu_css_cfg_gdc_table(map->vaddr,
2210 css_pipe->aux_frames[a].bytesperline /
2211 css_pipe->aux_frames[a].bytesperpixel,
2212 css_pipe->aux_frames[a].height,
2213 css_pipe->rect[g].width,
2214 css_pipe->rect[g].height,
2215 css_pipe->rect[e].width,
2216 css_pipe->rect[e].height);
2217 }
2218 }
2219
2220 /* Get a new obgrid only if a new obgrid is given, or none yet */
2221 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2222 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2223 imgu_css_pool_get(&css_pipe->pool.obgrid);
2224 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2225 obgrid = map->vaddr;
2226
2227 /* Configure optical black level grid (obgrid) */
2228 if (set_params && set_params->use.obgrid_param)
2229 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2230 obgrid[i] = set_params->obgrid_param;
2231 else
2232 memset(obgrid, 0, obgrid_size);
2233 }
2234
2235 /* Configure parameter set info, queued to `queue_id' */
2236
2237 memset(param_set, 0, sizeof(*param_set));
2238 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2239 param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2240
2241 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2242 param_set->mem_map.dvs_6axis_params_y = map->daddr;
2243
2244 for (i = 0; i < stripes; i++) {
2245 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2246 param_set->mem_map.obgrid_tbl[i] =
2247 map->daddr + (obgrid_size / stripes) * i;
2248 }
2249
2250 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2251 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2252 param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2253 }
2254
2255 /* Then queue the new parameter buffer */
2256 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2257 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2258 if (r < 0)
2259 goto fail;
2260
2261 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2262 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2263 queue_id));
2264 if (r < 0)
2265 goto fail_no_put;
2266
2267 /* Finally dequeue all old parameter buffers */
2268
2269 do {
2270 u32 daddr;
2271
2272 r = imgu_css_dequeue_data(css, queue_id, &daddr);
2273 if (r == -EBUSY)
2274 break;
2275 if (r)
2276 goto fail_no_put;
2277 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2278 IMGU_ABI_EVENT_BUFFER_DEQUEUED
2279 (queue_id));
2280 if (r < 0) {
2281 dev_err(css->dev, "failed to queue parameter event\n");
2282 goto fail_no_put;
2283 }
2284 } while (1);
2285
2286 return 0;
2287
2288 fail:
2289 /*
2290 * A failure, most likely the parameter queue was full.
2291 * Return error but continue streaming. User can try submitting new
2292 * parameters again later.
2293 */
2294
2295 imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2296 if (acc)
2297 imgu_css_pool_put(&css_pipe->pool.acc);
2298 if (gdc)
2299 imgu_css_pool_put(&css_pipe->pool.gdc);
2300 if (obgrid)
2301 imgu_css_pool_put(&css_pipe->pool.obgrid);
2302 if (vmem0)
2303 imgu_css_pool_put(
2304 &css_pipe->pool.binary_params_p
2305 [IMGU_ABI_MEM_ISP_VMEM0]);
2306 if (dmem0)
2307 imgu_css_pool_put(
2308 &css_pipe->pool.binary_params_p
2309 [IMGU_ABI_MEM_ISP_DMEM0]);
2310
2311 fail_no_put:
2312 return r;
2313 }
2314
imgu_css_irq_ack(struct imgu_css * css)2315 int imgu_css_irq_ack(struct imgu_css *css)
2316 {
2317 static const int NUM_SWIRQS = 3;
2318 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2319 void __iomem *const base = css->base;
2320 u32 irq_status[IMGU_IRQCTRL_NUM];
2321 int i;
2322
2323 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2324
2325 writel(imgu_status, base + IMGU_REG_INT_STATUS);
2326 for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2327 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2328
2329 for (i = 0; i < NUM_SWIRQS; i++) {
2330 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2331 /* SP SW interrupt */
2332 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2333 bi->info.sp.output);
2334 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2335 bi->info.sp.output + 4 + 4 * i);
2336
2337 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2338 __func__, i, cnt, val);
2339 }
2340 }
2341
2342 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2343 if (irq_status[i]) {
2344 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2345 /* Wait for write to complete */
2346 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2347 }
2348
2349 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2350 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2351 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2352
2353 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2354 return -ENOMSG;
2355
2356 return 0;
2357 }
2358