1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Support for Intel Camera Imaging ISP subsystem.
4 * Copyright (c) 2010 - 2016, Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 */
15
16 #include "isp.h"
17 #include "vmem.h"
18 #include "vmem_local.h"
19
20 #if !defined(HRT_MEMORY_ACCESS)
21 #include "ia_css_device_access.h"
22 #endif
23 #include "assert_support.h"
24
25 typedef unsigned long long hive_uedge;
26 typedef hive_uedge *hive_wide;
27
28 /* Copied from SDK: sim_semantics.c */
29
30 /* subword bits move like this: MSB[____xxxx____]LSB -> MSB[00000000xxxx]LSB */
31 #define SUBWORD(w, start, end) (((w) & (((1ULL << ((end) - 1)) - 1) << 1 | 1)) >> (start))
32
33 /* inverse subword bits move like this: MSB[xxxx____xxxx]LSB -> MSB[xxxx0000xxxx]LSB */
34 #define INV_SUBWORD(w, start, end) ((w) & (~(((1ULL << ((end) - 1)) - 1) << 1 | 1) | ((1ULL << (start)) - 1)))
35
36 #define uedge_bits (8 * sizeof(hive_uedge))
37 #define move_lower_bits(target, target_bit, src, src_bit) move_subword(target, target_bit, src, 0, src_bit)
38 #define move_upper_bits(target, target_bit, src, src_bit) move_subword(target, target_bit, src, src_bit, uedge_bits)
39 #define move_word(target, target_bit, src) move_subword(target, target_bit, src, 0, uedge_bits)
40
41 static void
move_subword(hive_uedge * target,unsigned int target_bit,hive_uedge src,unsigned int src_start,unsigned int src_end)42 move_subword(
43 hive_uedge *target,
44 unsigned int target_bit,
45 hive_uedge src,
46 unsigned int src_start,
47 unsigned int src_end)
48 {
49 unsigned int start_elem = target_bit / uedge_bits;
50 unsigned int start_bit = target_bit % uedge_bits;
51 unsigned int subword_width = src_end - src_start;
52
53 hive_uedge src_subword = SUBWORD(src, src_start, src_end);
54
55 if (subword_width + start_bit > uedge_bits) { /* overlap */
56 hive_uedge old_val1;
57 hive_uedge old_val0 = INV_SUBWORD(target[start_elem], start_bit, uedge_bits);
58
59 target[start_elem] = old_val0 | (src_subword << start_bit);
60 old_val1 = INV_SUBWORD(target[start_elem + 1], 0,
61 subword_width + start_bit - uedge_bits);
62 target[start_elem + 1] = old_val1 | (src_subword >> (uedge_bits - start_bit));
63 } else {
64 hive_uedge old_val = INV_SUBWORD(target[start_elem], start_bit,
65 start_bit + subword_width);
66
67 target[start_elem] = old_val | (src_subword << start_bit);
68 }
69 }
70
71 static void
hive_sim_wide_unpack(hive_wide vector,hive_wide elem,hive_uint elem_bits,hive_uint index)72 hive_sim_wide_unpack(
73 hive_wide vector,
74 hive_wide elem,
75 hive_uint elem_bits,
76 hive_uint index)
77 {
78 /* pointers into wide_type: */
79 unsigned int start_elem = (elem_bits * index) / uedge_bits;
80 unsigned int start_bit = (elem_bits * index) % uedge_bits;
81 unsigned int end_elem = (elem_bits * (index + 1) - 1) / uedge_bits;
82 unsigned int end_bit = ((elem_bits * (index + 1) - 1) % uedge_bits) + 1;
83
84 if (elem_bits == uedge_bits) {
85 /* easy case for speedup: */
86 elem[0] = vector[index];
87 } else if (start_elem == end_elem) {
88 /* only one (<=64 bits) element needs to be (partly) copied: */
89 move_subword(elem, 0, vector[start_elem], start_bit, end_bit);
90 } else {
91 /* general case: handles edge spanning cases (includes >64bit elements) */
92 unsigned int bits_written = 0;
93 unsigned int i;
94
95 move_upper_bits(elem, bits_written, vector[start_elem], start_bit);
96 bits_written += (64 - start_bit);
97 for (i = start_elem + 1; i < end_elem; i++) {
98 move_word(elem, bits_written, vector[i]);
99 bits_written += uedge_bits;
100 }
101 move_lower_bits(elem, bits_written, vector[end_elem], end_bit);
102 }
103 }
104
105 static void
hive_sim_wide_pack(hive_wide vector,hive_wide elem,hive_uint elem_bits,hive_uint index)106 hive_sim_wide_pack(
107 hive_wide vector,
108 hive_wide elem,
109 hive_uint elem_bits,
110 hive_uint index)
111 {
112 /* pointers into wide_type: */
113 unsigned int start_elem = (elem_bits * index) / uedge_bits;
114
115 /* easy case for speedup: */
116 if (elem_bits == uedge_bits) {
117 vector[start_elem] = elem[0];
118 } else if (elem_bits > uedge_bits) {
119 unsigned int bits_to_write = elem_bits;
120 unsigned int start_bit = elem_bits * index;
121 unsigned int i = 0;
122
123 for (; bits_to_write > uedge_bits;
124 bits_to_write -= uedge_bits, i++, start_bit += uedge_bits) {
125 move_word(vector, start_bit, elem[i]);
126 }
127 move_lower_bits(vector, start_bit, elem[i], bits_to_write);
128 } else {
129 /* only one element needs to be (partly) copied: */
130 move_lower_bits(vector, elem_bits * index, elem[0], elem_bits);
131 }
132 }
133
load_vector(const isp_ID_t ID,t_vmem_elem * to,const t_vmem_elem * from)134 static void load_vector(
135 const isp_ID_t ID,
136 t_vmem_elem *to,
137 const t_vmem_elem *from)
138 {
139 unsigned int i;
140 hive_uedge *data;
141 unsigned int size = sizeof(short) * ISP_NWAY;
142
143 VMEM_ARRAY(v, 2 * ISP_NWAY); /* Need 2 vectors to work around vmem hss bug */
144 assert(ISP_BAMEM_BASE[ID] != (hrt_address) - 1);
145 #if !defined(HRT_MEMORY_ACCESS)
146 ia_css_device_load(ISP_BAMEM_BASE[ID] + (unsigned long)from, &v[0][0], size);
147 #else
148 hrt_master_port_load(ISP_BAMEM_BASE[ID] + (unsigned long)from, &v[0][0], size);
149 #endif
150 data = (hive_uedge *)v;
151 for (i = 0; i < ISP_NWAY; i++) {
152 hive_uedge elem = 0;
153
154 hive_sim_wide_unpack(data, &elem, ISP_VEC_ELEMBITS, i);
155 to[i] = elem;
156 }
157 udelay(1); /* Spend at least 1 cycles per vector */
158 }
159
store_vector(const isp_ID_t ID,t_vmem_elem * to,const t_vmem_elem * from)160 static void store_vector(
161 const isp_ID_t ID,
162 t_vmem_elem *to,
163 const t_vmem_elem *from)
164 {
165 unsigned int i;
166 unsigned int size = sizeof(short) * ISP_NWAY;
167
168 VMEM_ARRAY(v, 2 * ISP_NWAY); /* Need 2 vectors to work around vmem hss bug */
169 //load_vector (&v[1][0], &to[ISP_NWAY]); /* Fetch the next vector, since it will be overwritten. */
170 hive_uedge *data = (hive_uedge *)v;
171
172 for (i = 0; i < ISP_NWAY; i++) {
173 hive_sim_wide_pack(data, (hive_wide)&from[i], ISP_VEC_ELEMBITS, i);
174 }
175 assert(ISP_BAMEM_BASE[ID] != (hrt_address) - 1);
176 #if !defined(HRT_MEMORY_ACCESS)
177 ia_css_device_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size);
178 #else
179 //hrt_mem_store (ISP, VMEM, (unsigned)to, &v, siz); /* This will overwrite the next vector as well */
180 hrt_master_port_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size);
181 #endif
182 udelay(1); /* Spend at least 1 cycles per vector */
183 }
184
isp_vmem_load(const isp_ID_t ID,const t_vmem_elem * from,t_vmem_elem * to,unsigned int elems)185 void isp_vmem_load(
186 const isp_ID_t ID,
187 const t_vmem_elem *from,
188 t_vmem_elem *to,
189 unsigned int elems) /* In t_vmem_elem */
190 {
191 unsigned int c;
192 const t_vmem_elem *vp = from;
193
194 assert(ID < N_ISP_ID);
195 assert((unsigned long)from % ISP_VEC_ALIGN == 0);
196 assert(elems % ISP_NWAY == 0);
197 for (c = 0; c < elems; c += ISP_NWAY) {
198 load_vector(ID, &to[c], vp);
199 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN);
200 }
201 }
202
isp_vmem_store(const isp_ID_t ID,t_vmem_elem * to,const t_vmem_elem * from,unsigned int elems)203 void isp_vmem_store(
204 const isp_ID_t ID,
205 t_vmem_elem *to,
206 const t_vmem_elem *from,
207 unsigned int elems) /* In t_vmem_elem */
208 {
209 unsigned int c;
210 t_vmem_elem *vp = to;
211
212 assert(ID < N_ISP_ID);
213 assert((unsigned long)to % ISP_VEC_ALIGN == 0);
214 assert(elems % ISP_NWAY == 0);
215 for (c = 0; c < elems; c += ISP_NWAY) {
216 store_vector(ID, vp, &from[c]);
217 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN);
218 }
219 }
220
isp_vmem_2d_load(const isp_ID_t ID,const t_vmem_elem * from,t_vmem_elem * to,unsigned int height,unsigned int width,unsigned int stride_to,unsigned stride_from)221 void isp_vmem_2d_load(
222 const isp_ID_t ID,
223 const t_vmem_elem *from,
224 t_vmem_elem *to,
225 unsigned int height,
226 unsigned int width,
227 unsigned int stride_to, /* In t_vmem_elem */
228
229 unsigned stride_from /* In t_vmem_elem */)
230 {
231 unsigned int h;
232
233 assert(ID < N_ISP_ID);
234 assert((unsigned long)from % ISP_VEC_ALIGN == 0);
235 assert(width % ISP_NWAY == 0);
236 assert(stride_from % ISP_NWAY == 0);
237 for (h = 0; h < height; h++) {
238 unsigned int c;
239 const t_vmem_elem *vp = from;
240
241 for (c = 0; c < width; c += ISP_NWAY) {
242 load_vector(ID, &to[stride_to * h + c], vp);
243 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN);
244 }
245 from = (const t_vmem_elem *)((const char *)from + stride_from / ISP_NWAY *
246 ISP_VEC_ALIGN);
247 }
248 }
249
isp_vmem_2d_store(const isp_ID_t ID,t_vmem_elem * to,const t_vmem_elem * from,unsigned int height,unsigned int width,unsigned int stride_to,unsigned stride_from)250 void isp_vmem_2d_store(
251 const isp_ID_t ID,
252 t_vmem_elem *to,
253 const t_vmem_elem *from,
254 unsigned int height,
255 unsigned int width,
256 unsigned int stride_to, /* In t_vmem_elem */
257
258 unsigned stride_from /* In t_vmem_elem */)
259 {
260 unsigned int h;
261
262 assert(ID < N_ISP_ID);
263 assert((unsigned long)to % ISP_VEC_ALIGN == 0);
264 assert(width % ISP_NWAY == 0);
265 assert(stride_to % ISP_NWAY == 0);
266 for (h = 0; h < height; h++) {
267 unsigned int c;
268 t_vmem_elem *vp = to;
269
270 for (c = 0; c < width; c += ISP_NWAY) {
271 store_vector(ID, vp, &from[stride_from * h + c]);
272 vp = (t_vmem_elem *)((char *)vp + ISP_VEC_ALIGN);
273 }
274 to = (t_vmem_elem *)((char *)to + stride_to / ISP_NWAY * ISP_VEC_ALIGN);
275 }
276 }
277