1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_sdma.h"
27 #include "amdgpu_ras.h"
28
29 #define AMDGPU_CSA_SDMA_SIZE 64
30 /* SDMA CSA reside in the 3rd page of CSA */
31 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
32
33 /*
34 * GPU SDMA IP block helpers function.
35 */
36
amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring * ring)37 struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
38 {
39 struct amdgpu_device *adev = ring->adev;
40 int i;
41
42 for (i = 0; i < adev->sdma.num_instances; i++)
43 if (ring == &adev->sdma.instance[i].ring ||
44 ring == &adev->sdma.instance[i].page)
45 return &adev->sdma.instance[i];
46
47 return NULL;
48 }
49
amdgpu_sdma_get_index_from_ring(struct amdgpu_ring * ring,uint32_t * index)50 int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
51 {
52 struct amdgpu_device *adev = ring->adev;
53 int i;
54
55 for (i = 0; i < adev->sdma.num_instances; i++) {
56 if (ring == &adev->sdma.instance[i].ring ||
57 ring == &adev->sdma.instance[i].page) {
58 *index = i;
59 return 0;
60 }
61 }
62
63 return -EINVAL;
64 }
65
amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring * ring,unsigned vmid)66 uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
67 unsigned vmid)
68 {
69 struct amdgpu_device *adev = ring->adev;
70 uint64_t csa_mc_addr;
71 uint32_t index = 0;
72 int r;
73
74 /* don't enable OS preemption on SDMA under SRIOV */
75 if (amdgpu_sriov_vf(adev) || vmid == 0 || !amdgpu_mcbp)
76 return 0;
77
78 if (ring->is_mes_queue) {
79 uint32_t offset = 0;
80
81 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
82 sdma[ring->idx].sdma_meta_data);
83 csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
84 } else {
85 r = amdgpu_sdma_get_index_from_ring(ring, &index);
86
87 if (r || index > 31)
88 csa_mc_addr = 0;
89 else
90 csa_mc_addr = amdgpu_csa_vaddr(adev) +
91 AMDGPU_CSA_SDMA_OFFSET +
92 index * AMDGPU_CSA_SDMA_SIZE;
93 }
94
95 return csa_mc_addr;
96 }
97
amdgpu_sdma_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)98 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
99 struct ras_common_if *ras_block)
100 {
101 int r, i;
102
103 r = amdgpu_ras_block_late_init(adev, ras_block);
104 if (r)
105 return r;
106
107 if (amdgpu_ras_is_supported(adev, ras_block->block)) {
108 for (i = 0; i < adev->sdma.num_instances; i++) {
109 r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
110 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
111 if (r)
112 goto late_fini;
113 }
114 }
115
116 return 0;
117
118 late_fini:
119 amdgpu_ras_block_late_fini(adev, ras_block);
120 return r;
121 }
122
amdgpu_sdma_process_ras_data_cb(struct amdgpu_device * adev,void * err_data,struct amdgpu_iv_entry * entry)123 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
124 void *err_data,
125 struct amdgpu_iv_entry *entry)
126 {
127 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
128
129 if (amdgpu_sriov_vf(adev))
130 return AMDGPU_RAS_SUCCESS;
131
132 amdgpu_ras_reset_gpu(adev);
133
134 return AMDGPU_RAS_SUCCESS;
135 }
136
amdgpu_sdma_process_ecc_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)137 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
138 struct amdgpu_irq_src *source,
139 struct amdgpu_iv_entry *entry)
140 {
141 struct ras_common_if *ras_if = adev->sdma.ras_if;
142 struct ras_dispatch_if ih_data = {
143 .entry = entry,
144 };
145
146 if (!ras_if)
147 return 0;
148
149 ih_data.head = *ras_if;
150
151 amdgpu_ras_interrupt_dispatch(adev, &ih_data);
152 return 0;
153 }
154
amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance * sdma_inst)155 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
156 {
157 int err = 0;
158 uint16_t version_major;
159 const struct common_firmware_header *header = NULL;
160 const struct sdma_firmware_header_v1_0 *hdr;
161 const struct sdma_firmware_header_v2_0 *hdr_v2;
162
163 err = amdgpu_ucode_validate(sdma_inst->fw);
164 if (err)
165 return err;
166
167 header = (const struct common_firmware_header *)
168 sdma_inst->fw->data;
169 version_major = le16_to_cpu(header->header_version_major);
170
171 switch (version_major) {
172 case 1:
173 hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
174 sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
175 sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
176 break;
177 case 2:
178 hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
179 sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
180 sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
181 break;
182 default:
183 return -EINVAL;
184 }
185
186 if (sdma_inst->feature_version >= 20)
187 sdma_inst->burst_nop = true;
188
189 return 0;
190 }
191
amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device * adev,bool duplicate)192 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
193 bool duplicate)
194 {
195 int i;
196
197 for (i = 0; i < adev->sdma.num_instances; i++) {
198 release_firmware(adev->sdma.instance[i].fw);
199 if (duplicate)
200 break;
201 }
202
203 memset((void *)adev->sdma.instance, 0,
204 sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
205 }
206
amdgpu_sdma_init_microcode(struct amdgpu_device * adev,char * fw_name,u32 instance,bool duplicate)207 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
208 char *fw_name, u32 instance,
209 bool duplicate)
210 {
211 struct amdgpu_firmware_info *info = NULL;
212 const struct common_firmware_header *header = NULL;
213 int err = 0, i;
214 const struct sdma_firmware_header_v2_0 *sdma_hdr;
215 uint16_t version_major;
216
217 err = request_firmware(&adev->sdma.instance[instance].fw, fw_name, adev->dev);
218 if (err)
219 goto out;
220
221 header = (const struct common_firmware_header *)
222 adev->sdma.instance[instance].fw->data;
223 version_major = le16_to_cpu(header->header_version_major);
224
225 if ((duplicate && instance) || (!duplicate && version_major > 1)) {
226 err = -EINVAL;
227 goto out;
228 }
229
230 err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
231 if (err)
232 goto out;
233
234 if (duplicate) {
235 for (i = 1; i < adev->sdma.num_instances; i++)
236 memcpy((void *)&adev->sdma.instance[i],
237 (void *)&adev->sdma.instance[0],
238 sizeof(struct amdgpu_sdma_instance));
239 }
240
241 if (amdgpu_sriov_vf(adev))
242 return 0;
243
244 DRM_DEBUG("psp_load == '%s'\n",
245 adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
246
247 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
248 switch (version_major) {
249 case 1:
250 for (i = 0; i < adev->sdma.num_instances; i++) {
251 if (!duplicate && (instance != i))
252 continue;
253 else {
254 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
255 info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
256 info->fw = adev->sdma.instance[i].fw;
257 adev->firmware.fw_size +=
258 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
259 }
260 }
261 break;
262 case 2:
263 sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
264 adev->sdma.instance[0].fw->data;
265 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
266 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
267 info->fw = adev->sdma.instance[0].fw;
268 adev->firmware.fw_size +=
269 ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
270 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
271 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
272 info->fw = adev->sdma.instance[0].fw;
273 adev->firmware.fw_size +=
274 ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
275 break;
276 default:
277 err = -EINVAL;
278 }
279 }
280
281 out:
282 if (err) {
283 DRM_ERROR("SDMA: Failed to init firmware \"%s\"\n", fw_name);
284 amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
285 }
286 return err;
287 }
288
amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device * adev)289 void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev)
290 {
291 struct amdgpu_ring *sdma;
292 int i;
293
294 for (i = 0; i < adev->sdma.num_instances; i++) {
295 if (adev->sdma.has_page_queue) {
296 sdma = &adev->sdma.instance[i].page;
297 if (adev->mman.buffer_funcs_ring == sdma) {
298 amdgpu_ttm_set_buffer_funcs_status(adev, false);
299 break;
300 }
301 }
302 sdma = &adev->sdma.instance[i].ring;
303 if (adev->mman.buffer_funcs_ring == sdma) {
304 amdgpu_ttm_set_buffer_funcs_status(adev, false);
305 break;
306 }
307 }
308 }
309