373 static void cobalt_newton_step(struct cobalt_vars *vars)  in cobalt_newton_step()  argument
378 	invsqrt = vars->rec_inv_sqrt;  in cobalt_newton_step()
380 	val = (3LL << 32) - ((u64)vars->count * invsqrt2);  in cobalt_newton_step()
385 	vars->rec_inv_sqrt = val;  in cobalt_newton_step()
388 static void cobalt_invsqrt(struct cobalt_vars *vars)  in cobalt_invsqrt()  argument
390 	if (vars->count < REC_INV_SQRT_CACHE)  in cobalt_invsqrt()
391 		vars->rec_inv_sqrt = cobalt_rec_inv_sqrt_cache[vars->count];  in cobalt_invsqrt()
393 		cobalt_newton_step(vars);  in cobalt_invsqrt()
424 static void cobalt_vars_init(struct cobalt_vars *vars)  in cobalt_vars_init()  argument
426 	memset(vars, 0, sizeof(*vars));  in cobalt_vars_init()
449 static bool cobalt_queue_full(struct cobalt_vars *vars,  in cobalt_queue_full()  argument
455 	if (ktime_to_ns(ktime_sub(now, vars->blue_timer)) > p->target) {  in cobalt_queue_full()
456 		up = !vars->p_drop;  in cobalt_queue_full()
457 		vars->p_drop += p->p_inc;  in cobalt_queue_full()
458 		if (vars->p_drop < p->p_inc)  in cobalt_queue_full()
459 			vars->p_drop = ~0;  in cobalt_queue_full()
460 		vars->blue_timer = now;  in cobalt_queue_full()
462 	vars->dropping = true;  in cobalt_queue_full()
463 	vars->drop_next = now;  in cobalt_queue_full()
464 	if (!vars->count)  in cobalt_queue_full()
465 		vars->count = 1;  in cobalt_queue_full()
473 static bool cobalt_queue_empty(struct cobalt_vars *vars,  in cobalt_queue_empty()  argument
479 	if (vars->p_drop &&  in cobalt_queue_empty()
480 	    ktime_to_ns(ktime_sub(now, vars->blue_timer)) > p->target) {  in cobalt_queue_empty()
481 		if (vars->p_drop < p->p_dec)  in cobalt_queue_empty()
482 			vars->p_drop = 0;  in cobalt_queue_empty()
484 			vars->p_drop -= p->p_dec;  in cobalt_queue_empty()
485 		vars->blue_timer = now;  in cobalt_queue_empty()
486 		down = !vars->p_drop;  in cobalt_queue_empty()
488 	vars->dropping = false;  in cobalt_queue_empty()
490 	if (vars->count && ktime_to_ns(ktime_sub(now, vars->drop_next)) >= 0) {  in cobalt_queue_empty()
491 		vars->count--;  in cobalt_queue_empty()
492 		cobalt_invsqrt(vars);  in cobalt_queue_empty()
493 		vars->drop_next = cobalt_control(vars->drop_next,  in cobalt_queue_empty()
495 						 vars->rec_inv_sqrt);  in cobalt_queue_empty()
504 static bool cobalt_should_drop(struct cobalt_vars *vars,  in cobalt_should_drop()  argument
530 	schedule = ktime_sub(now, vars->drop_next);  in cobalt_should_drop()
534 	next_due = vars->count && ktime_to_ns(schedule) >= 0;  in cobalt_should_drop()
536 	vars->ecn_marked = false;  in cobalt_should_drop()
539 		if (!vars->dropping) {  in cobalt_should_drop()
540 			vars->dropping = true;  in cobalt_should_drop()
541 			vars->drop_next = cobalt_control(now,  in cobalt_should_drop()
543 							 vars->rec_inv_sqrt);  in cobalt_should_drop()
545 		if (!vars->count)  in cobalt_should_drop()
546 			vars->count = 1;  in cobalt_should_drop()
547 	} else if (vars->dropping) {  in cobalt_should_drop()
548 		vars->dropping = false;  in cobalt_should_drop()
551 	if (next_due && vars->dropping) {  in cobalt_should_drop()
553 		drop = !(vars->ecn_marked = INET_ECN_set_ce(skb));  in cobalt_should_drop()
555 		vars->count++;  in cobalt_should_drop()
556 		if (!vars->count)  in cobalt_should_drop()
557 			vars->count--;  in cobalt_should_drop()
558 		cobalt_invsqrt(vars);  in cobalt_should_drop()
559 		vars->drop_next = cobalt_control(vars->drop_next,  in cobalt_should_drop()
561 						 vars->rec_inv_sqrt);  in cobalt_should_drop()
562 		schedule = ktime_sub(now, vars->drop_next);  in cobalt_should_drop()
565 			vars->count--;  in cobalt_should_drop()
566 			cobalt_invsqrt(vars);  in cobalt_should_drop()
567 			vars->drop_next = cobalt_control(vars->drop_next,  in cobalt_should_drop()
569 							 vars->rec_inv_sqrt);  in cobalt_should_drop()
570 			schedule = ktime_sub(now, vars->drop_next);  in cobalt_should_drop()
571 			next_due = vars->count && ktime_to_ns(schedule) >= 0;  in cobalt_should_drop()
576 	if (vars->p_drop)  in cobalt_should_drop()
577 		drop |= (get_random_u32() < vars->p_drop);  in cobalt_should_drop()
580 	if (!vars->count)  in cobalt_should_drop()
581 		vars->drop_next = ktime_add_ns(now, p->interval);  in cobalt_should_drop()
583 		vars->drop_next = now;  in cobalt_should_drop()