387 static struct rcu_torture_ops *cur_ops;  variable
419 		started = cur_ops->get_gp_seq();  in rcu_read_delay()
425 		completed = cur_ops->get_gp_seq();  in rcu_read_delay()
426 		do_trace_rcu_torture_read(cur_ops->name, NULL, ts,  in rcu_read_delay()
503 		cur_ops->deferred_free(rp);  in rcu_torture_cb()
978 	if (!cur_ops->gp_diff)  in rcutorture_seq_diff()
980 	return cur_ops->gp_diff(new, old);  in rcutorture_seq_diff()
1030 		if (cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost_failed()
1032 		if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {  in rcu_torture_boost_failed()
1042 		if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {  in rcu_torture_boost_failed()
1045 			cur_ops->gp_kthread_dbg();  in rcu_torture_boost_failed()
1047 			gp_done = cur_ops->poll_gp_state(gp_state);  in rcu_torture_boost_failed()
1054 	} else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) {  in rcu_torture_boost_failed()
1095 			if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost()
1098 			if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) {  in rcu_torture_boost()
1099 				gp_state = cur_ops->start_gp_poll();  in rcu_torture_boost()
1108 				if (cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost()
1116 		if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost()
1175 			cur_ops->fqs();  in rcu_torture_fqs()
1224 	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {  in rcu_torture_write_types()
1227 	} else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {  in rcu_torture_write_types()
1230 	if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {  in rcu_torture_write_types()
1233 	} else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {  in rcu_torture_write_types()
1236 	if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {  in rcu_torture_write_types()
1239 	} else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {  in rcu_torture_write_types()
1242 	if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {  in rcu_torture_write_types()
1246 		   (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {  in rcu_torture_write_types()
1249 	if (gp_exp1 && cur_ops->exp_sync) {  in rcu_torture_write_types()
1252 	} else if (gp_exp && !cur_ops->exp_sync) {  in rcu_torture_write_types()
1255 	if (gp_normal1 && cur_ops->deferred_free) {  in rcu_torture_write_types()
1258 	} else if (gp_normal && !cur_ops->deferred_free) {  in rcu_torture_write_types()
1261 	if (gp_poll1 && cur_ops->start_gp_poll && cur_ops->poll_gp_state) {  in rcu_torture_write_types()
1264 	} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {  in rcu_torture_write_types()
1267 	if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {  in rcu_torture_write_types()
1270 	} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {  in rcu_torture_write_types()
1273 	if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {  in rcu_torture_write_types()
1276 	} else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {  in rcu_torture_write_types()
1279 	if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {  in rcu_torture_write_types()
1283 		   (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {  in rcu_torture_write_types()
1286 	if (gp_sync1 && cur_ops->sync) {  in rcu_torture_write_types()
1289 	} else if (gp_sync && !cur_ops->sync) {  in rcu_torture_write_types()
1307 	dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);  in do_rtws_sync()
1308 	dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);  in do_rtws_sync()
1312 		cookie = cur_ops->get_gp_state();  in do_rtws_sync()
1314 		cur_ops->get_gp_state_full(&cookie_full);  in do_rtws_sync()
1315 	if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))  in do_rtws_sync()
1318 	WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),  in do_rtws_sync()
1321 	WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),  in do_rtws_sync()
1355 			 torture_type, cur_ops->name);  in rcu_torture_writer()
1392 			if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {  in rcu_torture_writer()
1393 				idx = cur_ops->readlock();  in rcu_torture_writer()
1394 				cookie = cur_ops->get_gp_state();  in rcu_torture_writer()
1395 				WARN_ONCE(cur_ops->poll_gp_state(cookie),  in rcu_torture_writer()
1400 					  cookie, cur_ops->get_gp_state());  in rcu_torture_writer()
1401 				if (cur_ops->get_gp_completed) {  in rcu_torture_writer()
1402 					cookie = cur_ops->get_gp_completed();  in rcu_torture_writer()
1403 					WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));  in rcu_torture_writer()
1405 				cur_ops->readunlock(idx);  in rcu_torture_writer()
1407 			if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {  in rcu_torture_writer()
1408 				idx = cur_ops->readlock();  in rcu_torture_writer()
1409 				cur_ops->get_gp_state_full(&cookie_full);  in rcu_torture_writer()
1410 				WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),  in rcu_torture_writer()
1416 				if (cur_ops->get_gp_completed_full) {  in rcu_torture_writer()
1417 					cur_ops->get_gp_completed_full(&cookie_full);  in rcu_torture_writer()
1418 					WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));  in rcu_torture_writer()
1420 				cur_ops->readunlock(idx);  in rcu_torture_writer()
1425 				cur_ops->deferred_free(old_rp);  in rcu_torture_writer()
1429 				do_rtws_sync(&rand, cur_ops->exp_sync);  in rcu_torture_writer()
1434 				gp_snap = cur_ops->get_gp_state();  in rcu_torture_writer()
1437 				cur_ops->cond_sync(gp_snap);  in rcu_torture_writer()
1442 				gp_snap = cur_ops->get_gp_state_exp();  in rcu_torture_writer()
1445 				cur_ops->cond_sync_exp(gp_snap);  in rcu_torture_writer()
1450 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_writer()
1453 				cur_ops->cond_sync_full(&gp_snap_full);  in rcu_torture_writer()
1458 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_writer()
1461 				cur_ops->cond_sync_exp_full(&gp_snap_full);  in rcu_torture_writer()
1466 				gp_snap = cur_ops->start_gp_poll();  in rcu_torture_writer()
1468 				while (!cur_ops->poll_gp_state(gp_snap))  in rcu_torture_writer()
1475 				cur_ops->start_gp_poll_full(&gp_snap_full);  in rcu_torture_writer()
1477 				while (!cur_ops->poll_gp_state_full(&gp_snap_full))  in rcu_torture_writer()
1484 				gp_snap = cur_ops->start_gp_poll_exp();  in rcu_torture_writer()
1486 				while (!cur_ops->poll_gp_state_exp(gp_snap))  in rcu_torture_writer()
1493 				cur_ops->start_gp_poll_exp_full(&gp_snap_full);  in rcu_torture_writer()
1495 				while (!cur_ops->poll_gp_state_full(&gp_snap_full))  in rcu_torture_writer()
1502 				do_rtws_sync(&rand, cur_ops->sync);  in rcu_torture_writer()
1531 		    !cur_ops->slow_gps &&  in rcu_torture_writer()
1588 		if (cur_ops->cb_barrier != NULL &&  in rcu_torture_fakewriter()
1590 			cur_ops->cb_barrier();  in rcu_torture_fakewriter()
1596 				cur_ops->exp_sync();  in rcu_torture_fakewriter()
1599 				gp_snap = cur_ops->get_gp_state();  in rcu_torture_fakewriter()
1601 				cur_ops->cond_sync(gp_snap);  in rcu_torture_fakewriter()
1604 				gp_snap = cur_ops->get_gp_state_exp();  in rcu_torture_fakewriter()
1606 				cur_ops->cond_sync_exp(gp_snap);  in rcu_torture_fakewriter()
1609 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_fakewriter()
1611 				cur_ops->cond_sync_full(&gp_snap_full);  in rcu_torture_fakewriter()
1614 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_fakewriter()
1616 				cur_ops->cond_sync_exp_full(&gp_snap_full);  in rcu_torture_fakewriter()
1619 				gp_snap = cur_ops->start_gp_poll();  in rcu_torture_fakewriter()
1620 				while (!cur_ops->poll_gp_state(gp_snap)) {  in rcu_torture_fakewriter()
1626 				cur_ops->start_gp_poll_full(&gp_snap_full);  in rcu_torture_fakewriter()
1627 				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {  in rcu_torture_fakewriter()
1633 				gp_snap = cur_ops->start_gp_poll_exp();  in rcu_torture_fakewriter()
1634 				while (!cur_ops->poll_gp_state_exp(gp_snap)) {  in rcu_torture_fakewriter()
1640 				cur_ops->start_gp_poll_exp_full(&gp_snap_full);  in rcu_torture_fakewriter()
1641 				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {  in rcu_torture_fakewriter()
1647 				cur_ops->sync();  in rcu_torture_fakewriter()
1759 		idxnew1 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_1;  in rcutorture_one_extend()
1761 		idxnew2 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_2;  in rcutorture_one_extend()
1781 		cur_ops->readunlock((idxold2 >> RCUTORTURE_RDR_SHIFT_2) & 0x1);  in rcutorture_one_extend()
1788 		lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);  in rcutorture_one_extend()
1791 		cur_ops->readunlock((idxold1 >> RCUTORTURE_RDR_SHIFT_1) & 0x1);  in rcutorture_one_extend()
1800 		cur_ops->read_delay(trsp, rtrsp);  in rcutorture_one_extend()
1824 	mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;  in rcutorture_extend_mask_max()
1928 		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)  in rcu_torture_one_read()
1929 			cookie = cur_ops->get_gp_state();  in rcu_torture_one_read()
1930 		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)  in rcu_torture_one_read()
1931 			cur_ops->get_gp_state_full(&cookie_full);  in rcu_torture_one_read()
1933 	started = cur_ops->get_gp_seq();  in rcu_torture_one_read()
1936 				  !cur_ops->readlock_held || cur_ops->readlock_held());  in rcu_torture_one_read()
1952 	completed = cur_ops->get_gp_seq();  in rcu_torture_one_read()
1954 		do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,  in rcu_torture_one_read()
1967 		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)  in rcu_torture_one_read()
1968 			WARN_ONCE(cur_ops->poll_gp_state(cookie),  in rcu_torture_one_read()
1973 				  cookie, cur_ops->get_gp_state());  in rcu_torture_one_read()
1974 		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)  in rcu_torture_one_read()
1975 			WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),  in rcu_torture_one_read()
2013 	if (cur_ops->call) {  in rcu_torture_timer()
2017 			cur_ops->call(rhp, rcu_torture_timer_cb);  in rcu_torture_timer()
2038 	if (irqreader && cur_ops->irq_capable)  in rcu_torture_reader()
2042 		if (irqreader && cur_ops->irq_capable) {  in rcu_torture_reader()
2056 	if (irqreader && cur_ops->irq_capable) {  in rcu_torture_reader()
2206 	if (cur_ops->stats)  in rcu_torture_stats_print()
2207 		cur_ops->stats();  in rcu_torture_stats_print()
2214 		rcutorture_get_gp_data(cur_ops->ttype,  in rcu_torture_stats_print()
2216 		srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,  in rcu_torture_stats_print()
2228 		if (cur_ops->gp_kthread_dbg)  in rcu_torture_stats_print()
2229 			cur_ops->gp_kthread_dbg();  in rcu_torture_stats_print()
2303 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)  in rcu_torture_print_module_parms()  argument
2321 		 test_boost, cur_ops->can_boost,  in rcu_torture_print_module_parms()
2414 		idx = cur_ops->readlock();  in rcu_torture_stall()
2436 		cur_ops->readunlock(idx);  in rcu_torture_stall()
2472 	cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);  in rcu_torture_fwd_prog_cb()
2555 	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();  in rcu_torture_fwd_cb_cr()
2621 	if (!cur_ops->sync)  in rcu_torture_fwd_prog_nr()
2623 	if (cur_ops->call && cur_ops->cb_barrier) {  in rcu_torture_fwd_prog_nr()
2630 	cur_ops->sync(); /* Later readers see above write. */  in rcu_torture_fwd_prog_nr()
2633 		cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);  in rcu_torture_fwd_prog_nr()
2636 	gps = cur_ops->get_gp_seq();  in rcu_torture_fwd_prog_nr()
2637 	sd = cur_ops->stall_dur() + 1;  in rcu_torture_fwd_prog_nr()
2645 		idx = cur_ops->readlock();  in rcu_torture_fwd_prog_nr()
2647 		cur_ops->readunlock(idx);  in rcu_torture_fwd_prog_nr()
2657 		gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);  in rcu_torture_fwd_prog_nr()
2664 		cur_ops->sync(); /* Wait for running CB to complete. */  in rcu_torture_fwd_prog_nr()
2665 		pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_nr()
2666 		cur_ops->cb_barrier(); /* Wait for queued callbacks. */  in rcu_torture_fwd_prog_nr()
2697 	if (!cur_ops->call)  in rcu_torture_fwd_prog_cr()
2702 	cur_ops->sync(); /* Later readers see above write. */  in rcu_torture_fwd_prog_cr()
2713 	gps = cur_ops->get_gp_seq();  in rcu_torture_fwd_prog_cr()
2730 		} else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {  in rcu_torture_fwd_prog_cr()
2744 			cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);  in rcu_torture_fwd_prog_cr()
2755 	gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);  in rcu_torture_fwd_prog_cr()
2756 	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_cr()
2757 	cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */  in rcu_torture_fwd_prog_cr()
2809 	cur_ops->cb_barrier();  in rcutorture_oom_notify()
2814 	cur_ops->cb_barrier();  in rcutorture_oom_notify()
2863 		if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&  in rcu_torture_fwd_prog()
2897 	if ((!cur_ops->sync && !cur_ops->call) ||  in rcu_torture_fwd_prog_init()
2898 	    (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||  in rcu_torture_fwd_prog_init()
2899 	    cur_ops == &rcu_busted_ops) {  in rcu_torture_fwd_prog_init()
2974 	cur_ops->call(rhp, rcu_torture_barrier_cbf);  in rcu_torture_barrier1cb()
3003 			cur_ops->call(&rcu, rcu_torture_barrier_cbf);  in rcu_torture_barrier_cbs()
3008 	if (cur_ops->cb_barrier != NULL)  in rcu_torture_barrier_cbs()
3009 		cur_ops->cb_barrier();  in rcu_torture_barrier_cbs()
3034 		cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */  in rcu_torture_barrier()
3047 				cur_ops->cb_barrier();  in rcu_torture_barrier()
3072 	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {  in rcu_torture_barrier_init()
3075 			 torture_type, cur_ops->name);  in rcu_torture_barrier_init()
3125 	if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)  in rcu_torture_can_boost()
3127 	if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)  in rcu_torture_can_boost()
3243 		if (cur_ops->cb_barrier != NULL) {  in rcu_torture_cleanup()
3244 			pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);  in rcu_torture_cleanup()
3245 			cur_ops->cb_barrier();  in rcu_torture_cleanup()
3250 	if (!cur_ops) {  in rcu_torture_cleanup()
3256 	if (cur_ops->gp_kthread_dbg)  in rcu_torture_cleanup()
3257 		cur_ops->gp_kthread_dbg();  in rcu_torture_cleanup()
3289 	rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq);  in rcu_torture_cleanup()
3290 	srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq);  in rcu_torture_cleanup()
3292 		 cur_ops->name, (long)gp_seq, flags,  in rcu_torture_cleanup()
3303 	if (cur_ops->cb_barrier != NULL) {  in rcu_torture_cleanup()
3304 		pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);  in rcu_torture_cleanup()
3305 		cur_ops->cb_barrier();  in rcu_torture_cleanup()
3307 	if (cur_ops->cleanup != NULL)  in rcu_torture_cleanup()
3308 		cur_ops->cleanup();  in rcu_torture_cleanup()
3342 		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");  in rcu_torture_cleanup()
3344 		rcu_torture_print_module_parms(cur_ops,  in rcu_torture_cleanup()
3347 		rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");  in rcu_torture_cleanup()
3418 	if (cur_ops->sync && !(++n & 0xfff))  in rcutorture_sync()
3419 		cur_ops->sync();  in rcutorture_sync()
3441 		cur_ops = torture_ops[i];  in rcu_torture_init()
3442 		if (strcmp(torture_type, cur_ops->name) == 0)  in rcu_torture_init()
3453 		cur_ops = NULL;  in rcu_torture_init()
3456 	if (cur_ops->fqs == NULL && fqs_duration != 0) {  in rcu_torture_init()
3460 	if (cur_ops->init)  in rcu_torture_init()
3461 		cur_ops->init();  in rcu_torture_init()
3470 	rcu_torture_print_module_parms(cur_ops, "Start of test");  in rcu_torture_init()
3471 	rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq);  in rcu_torture_init()
3472 	srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq);  in rcu_torture_init()
3475 		 cur_ops->name, (long)gp_seq, flags);  in rcu_torture_init()
3588 		t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;  in rcu_torture_init()