Lines Matching refs:gts
70 struct gru_thread_state *gts = NULL; in gru_find_lock_gts() local
75 gts = gru_find_thread_state(vma, TSID(vaddr, vma)); in gru_find_lock_gts()
76 if (gts) in gru_find_lock_gts()
77 mutex_lock(>s->ts_ctxlock); in gru_find_lock_gts()
80 return gts; in gru_find_lock_gts()
87 struct gru_thread_state *gts = ERR_PTR(-EINVAL); in gru_alloc_locked_gts() local
94 gts = gru_alloc_thread_state(vma, TSID(vaddr, vma)); in gru_alloc_locked_gts()
95 if (IS_ERR(gts)) in gru_alloc_locked_gts()
97 mutex_lock(>s->ts_ctxlock); in gru_alloc_locked_gts()
99 return gts; in gru_alloc_locked_gts()
103 return gts; in gru_alloc_locked_gts()
109 static void gru_unlock_gts(struct gru_thread_state *gts) in gru_unlock_gts() argument
111 mutex_unlock(>s->ts_ctxlock); in gru_unlock_gts()
252 static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, in gru_vtop() argument
255 struct mm_struct *mm = gts->ts_mm; in gru_vtop()
310 struct gru_thread_state *gts, int atomic, in gru_preload_tlb() argument
332 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); in gru_preload_tlb()
338 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, in gru_preload_tlb()
356 struct gru_thread_state *gts, in gru_try_dropin() argument
361 unsigned char tlb_preload_count = gts->ts_tlb_preload_count; in gru_try_dropin()
410 if (atomic_read(>s->ts_gms->ms_range_active)) in gru_try_dropin()
413 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); in gru_try_dropin()
419 if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) { in gru_try_dropin()
420 gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift); in gru_try_dropin()
421 if (atomic || !gru_update_cch(gts)) { in gru_try_dropin()
422 gts->ts_force_cch_reload = 1; in gru_try_dropin()
428 gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe); in gru_try_dropin()
433 gts->ustats.tlbdropin++; in gru_try_dropin()
439 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid, in gru_try_dropin()
523 struct gru_thread_state *gts; in gru_intr() local
564 gts = gru->gs_gts[ctxnum]; in gru_intr()
567 if (!gts) { in gru_intr()
576 gts->ustats.fmm_tlbmiss++; in gru_intr()
577 if (!gts->ts_force_cch_reload && in gru_intr()
578 mmap_read_trylock(gts->ts_mm)) { in gru_intr()
579 gru_try_dropin(gru, gts, tfh, NULL); in gru_intr()
580 mmap_read_unlock(gts->ts_mm); in gru_intr()
613 static int gru_user_dropin(struct gru_thread_state *gts, in gru_user_dropin() argument
617 struct gru_mm_struct *gms = gts->ts_gms; in gru_user_dropin()
620 gts->ustats.upm_tlbmiss++; in gru_user_dropin()
625 ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb); in gru_user_dropin()
640 struct gru_thread_state *gts; in gru_handle_user_call_os() local
651 gts = gru_find_lock_gts(cb); in gru_handle_user_call_os()
652 if (!gts) in gru_handle_user_call_os()
654 …u_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts… in gru_handle_user_call_os()
656 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) in gru_handle_user_call_os()
659 gru_check_context_placement(gts); in gru_handle_user_call_os()
664 if (gts->ts_gru && gts->ts_force_cch_reload) { in gru_handle_user_call_os()
665 gts->ts_force_cch_reload = 0; in gru_handle_user_call_os()
666 gru_update_cch(gts); in gru_handle_user_call_os()
670 cbrnum = thread_cbr_number(gts, ucbnum); in gru_handle_user_call_os()
671 if (gts->ts_gru) { in gru_handle_user_call_os()
672 tfh = get_tfh_by_index(gts->ts_gru, cbrnum); in gru_handle_user_call_os()
673 cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr, in gru_handle_user_call_os()
674 gts->ts_ctxnum, ucbnum); in gru_handle_user_call_os()
675 ret = gru_user_dropin(gts, tfh, cbk); in gru_handle_user_call_os()
678 gru_unlock_gts(gts); in gru_handle_user_call_os()
690 struct gru_thread_state *gts; in gru_get_exception_detail() local
697 gts = gru_find_lock_gts(excdet.cb); in gru_get_exception_detail()
698 if (!gts) in gru_get_exception_detail()
701 …rudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts… in gru_get_exception_detail()
703 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { in gru_get_exception_detail()
705 } else if (gts->ts_gru) { in gru_get_exception_detail()
706 cbrnum = thread_cbr_number(gts, ucbnum); in gru_get_exception_detail()
707 cbe = get_cbe_by_index(gts->ts_gru, cbrnum); in gru_get_exception_detail()
722 gru_unlock_gts(gts); in gru_get_exception_detail()
739 struct gru_thread_state *gts; in gru_unload_all_contexts() local
749 gts = gru->gs_gts[ctxnum]; in gru_unload_all_contexts()
750 if (gts && mutex_trylock(>s->ts_ctxlock)) { in gru_unload_all_contexts()
752 gru_unload_context(gts, 1); in gru_unload_all_contexts()
753 mutex_unlock(>s->ts_ctxlock); in gru_unload_all_contexts()
764 struct gru_thread_state *gts; in gru_user_unload_context() local
776 gts = gru_find_lock_gts(req.gseg); in gru_user_unload_context()
777 if (!gts) in gru_user_unload_context()
780 if (gts->ts_gru) in gru_user_unload_context()
781 gru_unload_context(gts, 1); in gru_user_unload_context()
782 gru_unlock_gts(gts); in gru_user_unload_context()
793 struct gru_thread_state *gts; in gru_user_flush_tlb() local
804 gts = gru_find_lock_gts(req.gseg); in gru_user_flush_tlb()
805 if (!gts) in gru_user_flush_tlb()
808 gms = gts->ts_gms; in gru_user_flush_tlb()
809 gru_unlock_gts(gts); in gru_user_flush_tlb()
820 struct gru_thread_state *gts; in gru_get_gseg_statistics() local
831 gts = gru_find_lock_gts(req.gseg); in gru_get_gseg_statistics()
832 if (gts) { in gru_get_gseg_statistics()
833 memcpy(&req.stats, >s->ustats, sizeof(gts->ustats)); in gru_get_gseg_statistics()
834 gru_unlock_gts(gts); in gru_get_gseg_statistics()
836 memset(&req.stats, 0, sizeof(gts->ustats)); in gru_get_gseg_statistics()
851 struct gru_thread_state *gts; in gru_set_context_option() local
860 gts = gru_find_lock_gts(req.gseg); in gru_set_context_option()
861 if (!gts) { in gru_set_context_option()
862 gts = gru_alloc_locked_gts(req.gseg); in gru_set_context_option()
863 if (IS_ERR(gts)) in gru_set_context_option()
864 return PTR_ERR(gts); in gru_set_context_option()
875 gts->ts_user_blade_id = req.val1; in gru_set_context_option()
876 gts->ts_user_chiplet_id = req.val0; in gru_set_context_option()
877 gru_check_context_placement(gts); in gru_set_context_option()
882 gts->ts_tgid_owner = current->tgid; in gru_set_context_option()
886 gts->ts_cch_req_slice = req.val1 & 3; in gru_set_context_option()
891 gru_unlock_gts(gts); in gru_set_context_option()