21 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in _a6xx_check_idle()  local
24 	if (!a6xx_gmu_isidle(&a6xx_gpu->gmu))  in _a6xx_check_idle()
58 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in update_shadow_rptr()  local
61 	if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) {  in update_shadow_rptr()
63 		OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring)));  in update_shadow_rptr()
64 		OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring)));  in update_shadow_rptr()
102 static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,  in a6xx_set_pagetable()  argument
105 	bool sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1;  in a6xx_set_pagetable()
110 	if (ctx->seqno == a6xx_gpu->base.base.cur_ctx_seqno)  in a6xx_set_pagetable()
176 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_submit()  local
180 	a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);  in a6xx_submit()
595 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_set_hwcg()  local
596 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;  in a6xx_set_hwcg()
858 static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu,  in a6xx_ucode_check_version()  argument
861 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;  in a6xx_ucode_check_version()
893 			a6xx_gpu->has_whereami = true;  in a6xx_ucode_check_version()
924 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_ucode_init()  local
926 	if (!a6xx_gpu->sqe_bo) {  in a6xx_ucode_init()
927 		a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,  in a6xx_ucode_init()
928 			adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);  in a6xx_ucode_init()
930 		if (IS_ERR(a6xx_gpu->sqe_bo)) {  in a6xx_ucode_init()
931 			int ret = PTR_ERR(a6xx_gpu->sqe_bo);  in a6xx_ucode_init()
933 			a6xx_gpu->sqe_bo = NULL;  in a6xx_ucode_init()
940 		msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw");  in a6xx_ucode_init()
941 		if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) {  in a6xx_ucode_init()
942 			msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);  in a6xx_ucode_init()
943 			drm_gem_object_put(a6xx_gpu->sqe_bo);  in a6xx_ucode_init()
945 			a6xx_gpu->sqe_bo = NULL;  in a6xx_ucode_init()
951 		REG_A6XX_CP_SQE_INSTR_BASE+1, a6xx_gpu->sqe_iova);  in a6xx_ucode_init()
985 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in hw_init()  local
989 	a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);  in hw_init()
1166 	if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) {  in hw_init()
1167 		if (!a6xx_gpu->shadow_bo) {  in hw_init()
1168 			a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,  in hw_init()
1171 				gpu->aspace, &a6xx_gpu->shadow_bo,  in hw_init()
1172 				&a6xx_gpu->shadow_iova);  in hw_init()
1174 			if (IS_ERR(a6xx_gpu->shadow))  in hw_init()
1175 				return PTR_ERR(a6xx_gpu->shadow);  in hw_init()
1177 			msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");  in hw_init()
1182 			shadowptr(a6xx_gpu, gpu->rb[0]));  in hw_init()
1186 	a6xx_gpu->cur_ring = gpu->rb[0];  in hw_init()
1232 	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);  in hw_init()
1234 	if (a6xx_gpu->gmu.legacy) {  in hw_init()
1236 		a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);  in hw_init()
1245 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_hw_init()  local
1248 	mutex_lock(&a6xx_gpu->gmu.lock);  in a6xx_hw_init()
1250 	mutex_unlock(&a6xx_gpu->gmu.lock);  in a6xx_hw_init()
1268 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_recover()  local
1284 	a6xx_gpu->hung = true;  in a6xx_recover()
1293 	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);  in a6xx_recover()
1328 	a6xx_gpu->hung = false;  in a6xx_recover()
1485 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_fault_detect_irq()  local
1501 	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);  in a6xx_fault_detect_irq()
1554 static void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u32 or)  in a6xx_llc_rmw()  argument
1556 	return msm_rmw(a6xx_gpu->llc_mmio + (reg << 2), mask, or);  in a6xx_llc_rmw()
1559 static void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value)  in a6xx_llc_write()  argument
1561 	msm_writel(value, a6xx_gpu->llc_mmio + (reg << 2));  in a6xx_llc_write()
1564 static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu)  in a6xx_llc_deactivate()  argument
1566 	llcc_slice_deactivate(a6xx_gpu->llc_slice);  in a6xx_llc_deactivate()
1567 	llcc_slice_deactivate(a6xx_gpu->htw_llc_slice);  in a6xx_llc_deactivate()
1570 static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)  in a6xx_llc_activate()  argument
1572 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;  in a6xx_llc_activate()
1576 	if (IS_ERR(a6xx_gpu->llc_mmio))  in a6xx_llc_activate()
1579 	if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {  in a6xx_llc_activate()
1580 		u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);  in a6xx_llc_activate()
1598 	if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) {  in a6xx_llc_activate()
1599 		if (!a6xx_gpu->have_mmu500) {  in a6xx_llc_activate()
1600 			u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice);  in a6xx_llc_activate()
1614 	if (!a6xx_gpu->have_mmu500) {  in a6xx_llc_activate()
1615 		a6xx_llc_write(a6xx_gpu,  in a6xx_llc_activate()
1622 		a6xx_llc_rmw(a6xx_gpu,  in a6xx_llc_activate()
1630 static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)  in a6xx_llc_slices_destroy()  argument
1632 	llcc_slice_putd(a6xx_gpu->llc_slice);  in a6xx_llc_slices_destroy()
1633 	llcc_slice_putd(a6xx_gpu->htw_llc_slice);  in a6xx_llc_slices_destroy()
1637 		struct a6xx_gpu *a6xx_gpu)  in a6xx_llc_slices_init()  argument
1646 	a6xx_gpu->have_mmu500 = (phandle &&  in a6xx_llc_slices_init()
1650 	if (a6xx_gpu->have_mmu500)  in a6xx_llc_slices_init()
1651 		a6xx_gpu->llc_mmio = NULL;  in a6xx_llc_slices_init()
1653 		a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem");  in a6xx_llc_slices_init()
1655 	a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU);  in a6xx_llc_slices_init()
1656 	a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW);  in a6xx_llc_slices_init()
1658 	if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))  in a6xx_llc_slices_init()
1659 		a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL);  in a6xx_llc_slices_init()
1665 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_pm_resume()  local
1672 	mutex_lock(&a6xx_gpu->gmu.lock);  in a6xx_pm_resume()
1673 	ret = a6xx_gmu_resume(a6xx_gpu);  in a6xx_pm_resume()
1674 	mutex_unlock(&a6xx_gpu->gmu.lock);  in a6xx_pm_resume()
1680 	a6xx_llc_activate(a6xx_gpu);  in a6xx_pm_resume()
1688 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_pm_suspend()  local
1693 	a6xx_llc_deactivate(a6xx_gpu);  in a6xx_pm_suspend()
1697 	mutex_lock(&a6xx_gpu->gmu.lock);  in a6xx_pm_suspend()
1698 	ret = a6xx_gmu_stop(a6xx_gpu);  in a6xx_pm_suspend()
1699 	mutex_unlock(&a6xx_gpu->gmu.lock);  in a6xx_pm_suspend()
1703 	if (a6xx_gpu->shadow_bo)  in a6xx_pm_suspend()
1705 			a6xx_gpu->shadow[i] = 0;  in a6xx_pm_suspend()
1715 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_get_timestamp()  local
1717 	mutex_lock(&a6xx_gpu->gmu.lock);  in a6xx_get_timestamp()
1720 	a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);  in a6xx_get_timestamp()
1725 	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);  in a6xx_get_timestamp()
1727 	mutex_unlock(&a6xx_gpu->gmu.lock);  in a6xx_get_timestamp()
1735 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_active_ring()  local
1737 	return a6xx_gpu->cur_ring;  in a6xx_active_ring()
1743 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_destroy()  local
1745 	if (a6xx_gpu->sqe_bo) {  in a6xx_destroy()
1746 		msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);  in a6xx_destroy()
1747 		drm_gem_object_put(a6xx_gpu->sqe_bo);  in a6xx_destroy()
1750 	if (a6xx_gpu->shadow_bo) {  in a6xx_destroy()
1751 		msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace);  in a6xx_destroy()
1752 		drm_gem_object_put(a6xx_gpu->shadow_bo);  in a6xx_destroy()
1755 	a6xx_llc_slices_destroy(a6xx_gpu);  in a6xx_destroy()
1757 	a6xx_gmu_remove(a6xx_gpu);  in a6xx_destroy()
1761 	kfree(a6xx_gpu);  in a6xx_destroy()
1767 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_gpu_busy()  local
1773 	busy_cycles = gmu_read64(&a6xx_gpu->gmu,  in a6xx_gpu_busy()
1784 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_gpu_set_freq()  local
1786 	mutex_lock(&a6xx_gpu->gmu.lock);  in a6xx_gpu_set_freq()
1788 	mutex_unlock(&a6xx_gpu->gmu.lock);  in a6xx_gpu_set_freq()
1795 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_create_address_space()  local
1809 	if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))  in a6xx_create_address_space()
1853 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);  in a6xx_get_rptr()  local
1855 	if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami)  in a6xx_get_rptr()
1856 		return a6xx_gpu->shadow[ring->id];  in a6xx_get_rptr()
1988 	struct a6xx_gpu *a6xx_gpu;  in a6xx_gpu_init()  local
1993 	a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);  in a6xx_gpu_init()
1994 	if (!a6xx_gpu)  in a6xx_gpu_init()
1997 	adreno_gpu = &a6xx_gpu->base;  in a6xx_gpu_init()
2020 	a6xx_llc_slices_init(pdev, a6xx_gpu);  in a6xx_gpu_init()
2024 		a6xx_destroy(&(a6xx_gpu->base.base));  in a6xx_gpu_init()
2030 		a6xx_destroy(&(a6xx_gpu->base.base));  in a6xx_gpu_init()
2040 	ret = a6xx_gmu_init(a6xx_gpu, node);  in a6xx_gpu_init()
2043 		a6xx_destroy(&(a6xx_gpu->base.base));  in a6xx_gpu_init()