Lines Matching refs:gts
83 struct gru_thread_state *gts = NULL; in gru_find_lock_gts() local
88 gts = gru_find_thread_state(vma, TSID(vaddr, vma)); in gru_find_lock_gts()
89 if (gts) in gru_find_lock_gts()
90 mutex_lock(>s->ts_ctxlock); in gru_find_lock_gts()
93 return gts; in gru_find_lock_gts()
100 struct gru_thread_state *gts = ERR_PTR(-EINVAL); in gru_alloc_locked_gts() local
107 gts = gru_alloc_thread_state(vma, TSID(vaddr, vma)); in gru_alloc_locked_gts()
108 if (IS_ERR(gts)) in gru_alloc_locked_gts()
110 mutex_lock(>s->ts_ctxlock); in gru_alloc_locked_gts()
112 return gts; in gru_alloc_locked_gts()
116 return gts; in gru_alloc_locked_gts()
122 static void gru_unlock_gts(struct gru_thread_state *gts) in gru_unlock_gts() argument
124 mutex_unlock(>s->ts_ctxlock); in gru_unlock_gts()
261 static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, in gru_vtop() argument
264 struct mm_struct *mm = gts->ts_mm; in gru_vtop()
319 struct gru_thread_state *gts, int atomic, in gru_preload_tlb() argument
341 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); in gru_preload_tlb()
347 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, in gru_preload_tlb()
365 struct gru_thread_state *gts, in gru_try_dropin() argument
370 unsigned char tlb_preload_count = gts->ts_tlb_preload_count; in gru_try_dropin()
419 if (atomic_read(>s->ts_gms->ms_range_active)) in gru_try_dropin()
422 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); in gru_try_dropin()
428 if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) { in gru_try_dropin()
429 gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift); in gru_try_dropin()
430 if (atomic || !gru_update_cch(gts)) { in gru_try_dropin()
431 gts->ts_force_cch_reload = 1; in gru_try_dropin()
437 gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe); in gru_try_dropin()
442 gts->ustats.tlbdropin++; in gru_try_dropin()
448 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid, in gru_try_dropin()
532 struct gru_thread_state *gts; in gru_intr() local
573 gts = gru->gs_gts[ctxnum]; in gru_intr()
576 if (!gts) { in gru_intr()
585 gts->ustats.fmm_tlbmiss++; in gru_intr()
586 if (!gts->ts_force_cch_reload && in gru_intr()
587 down_read_trylock(>s->ts_mm->mmap_sem)) { in gru_intr()
588 gru_try_dropin(gru, gts, tfh, NULL); in gru_intr()
589 up_read(>s->ts_mm->mmap_sem); in gru_intr()
622 static int gru_user_dropin(struct gru_thread_state *gts, in gru_user_dropin() argument
626 struct gru_mm_struct *gms = gts->ts_gms; in gru_user_dropin()
629 gts->ustats.upm_tlbmiss++; in gru_user_dropin()
634 ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb); in gru_user_dropin()
649 struct gru_thread_state *gts; in gru_handle_user_call_os() local
660 gts = gru_find_lock_gts(cb); in gru_handle_user_call_os()
661 if (!gts) in gru_handle_user_call_os()
663 …u_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts… in gru_handle_user_call_os()
665 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) in gru_handle_user_call_os()
668 gru_check_context_placement(gts); in gru_handle_user_call_os()
673 if (gts->ts_gru && gts->ts_force_cch_reload) { in gru_handle_user_call_os()
674 gts->ts_force_cch_reload = 0; in gru_handle_user_call_os()
675 gru_update_cch(gts); in gru_handle_user_call_os()
679 cbrnum = thread_cbr_number(gts, ucbnum); in gru_handle_user_call_os()
680 if (gts->ts_gru) { in gru_handle_user_call_os()
681 tfh = get_tfh_by_index(gts->ts_gru, cbrnum); in gru_handle_user_call_os()
682 cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr, in gru_handle_user_call_os()
683 gts->ts_ctxnum, ucbnum); in gru_handle_user_call_os()
684 ret = gru_user_dropin(gts, tfh, cbk); in gru_handle_user_call_os()
687 gru_unlock_gts(gts); in gru_handle_user_call_os()
699 struct gru_thread_state *gts; in gru_get_exception_detail() local
706 gts = gru_find_lock_gts(excdet.cb); in gru_get_exception_detail()
707 if (!gts) in gru_get_exception_detail()
710 …rudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts… in gru_get_exception_detail()
712 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { in gru_get_exception_detail()
714 } else if (gts->ts_gru) { in gru_get_exception_detail()
715 cbrnum = thread_cbr_number(gts, ucbnum); in gru_get_exception_detail()
716 cbe = get_cbe_by_index(gts->ts_gru, cbrnum); in gru_get_exception_detail()
731 gru_unlock_gts(gts); in gru_get_exception_detail()
748 struct gru_thread_state *gts; in gru_unload_all_contexts() local
758 gts = gru->gs_gts[ctxnum]; in gru_unload_all_contexts()
759 if (gts && mutex_trylock(>s->ts_ctxlock)) { in gru_unload_all_contexts()
761 gru_unload_context(gts, 1); in gru_unload_all_contexts()
762 mutex_unlock(>s->ts_ctxlock); in gru_unload_all_contexts()
773 struct gru_thread_state *gts; in gru_user_unload_context() local
785 gts = gru_find_lock_gts(req.gseg); in gru_user_unload_context()
786 if (!gts) in gru_user_unload_context()
789 if (gts->ts_gru) in gru_user_unload_context()
790 gru_unload_context(gts, 1); in gru_user_unload_context()
791 gru_unlock_gts(gts); in gru_user_unload_context()
802 struct gru_thread_state *gts; in gru_user_flush_tlb() local
813 gts = gru_find_lock_gts(req.gseg); in gru_user_flush_tlb()
814 if (!gts) in gru_user_flush_tlb()
817 gms = gts->ts_gms; in gru_user_flush_tlb()
818 gru_unlock_gts(gts); in gru_user_flush_tlb()
829 struct gru_thread_state *gts; in gru_get_gseg_statistics() local
840 gts = gru_find_lock_gts(req.gseg); in gru_get_gseg_statistics()
841 if (gts) { in gru_get_gseg_statistics()
842 memcpy(&req.stats, >s->ustats, sizeof(gts->ustats)); in gru_get_gseg_statistics()
843 gru_unlock_gts(gts); in gru_get_gseg_statistics()
845 memset(&req.stats, 0, sizeof(gts->ustats)); in gru_get_gseg_statistics()
860 struct gru_thread_state *gts; in gru_set_context_option() local
869 gts = gru_find_lock_gts(req.gseg); in gru_set_context_option()
870 if (!gts) { in gru_set_context_option()
871 gts = gru_alloc_locked_gts(req.gseg); in gru_set_context_option()
872 if (IS_ERR(gts)) in gru_set_context_option()
873 return PTR_ERR(gts); in gru_set_context_option()
883 gts->ts_user_blade_id = req.val1; in gru_set_context_option()
884 gts->ts_user_chiplet_id = req.val0; in gru_set_context_option()
885 gru_check_context_placement(gts); in gru_set_context_option()
890 gts->ts_tgid_owner = current->tgid; in gru_set_context_option()
894 gts->ts_cch_req_slice = req.val1 & 3; in gru_set_context_option()
899 gru_unlock_gts(gts); in gru_set_context_option()