238 static int exynos5_counters_set_event(struct exynos5_dmc *dmc)  in exynos5_counters_set_event()  argument
242 	for (i = 0; i < dmc->num_counters; i++) {  in exynos5_counters_set_event()
243 		if (!dmc->counter[i])  in exynos5_counters_set_event()
245 		ret = devfreq_event_set_event(dmc->counter[i]);  in exynos5_counters_set_event()
252 static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)  in exynos5_counters_enable_edev()  argument
256 	for (i = 0; i < dmc->num_counters; i++) {  in exynos5_counters_enable_edev()
257 		if (!dmc->counter[i])  in exynos5_counters_enable_edev()
259 		ret = devfreq_event_enable_edev(dmc->counter[i]);  in exynos5_counters_enable_edev()
266 static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)  in exynos5_counters_disable_edev()  argument
270 	for (i = 0; i < dmc->num_counters; i++) {  in exynos5_counters_disable_edev()
271 		if (!dmc->counter[i])  in exynos5_counters_disable_edev()
273 		ret = devfreq_event_disable_edev(dmc->counter[i]);  in exynos5_counters_disable_edev()
288 static int find_target_freq_idx(struct exynos5_dmc *dmc,  in find_target_freq_idx()  argument
293 	for (i = dmc->opp_count - 1; i >= 0; i--)  in find_target_freq_idx()
294 		if (dmc->opp[i].freq_hz <= target_rate)  in find_target_freq_idx()
313 static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)  in exynos5_switch_timing_regs()  argument
318 	ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, &reg);  in exynos5_switch_timing_regs()
327 	regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);  in exynos5_switch_timing_regs()
339 static int exynos5_init_freq_table(struct exynos5_dmc *dmc,  in exynos5_init_freq_table()  argument
346 	ret = devm_pm_opp_of_add_table(dmc->dev);  in exynos5_init_freq_table()
348 		dev_err(dmc->dev, "Failed to get OPP table\n");  in exynos5_init_freq_table()
352 	dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);  in exynos5_init_freq_table()
354 	dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,  in exynos5_init_freq_table()
356 	if (!dmc->opp)  in exynos5_init_freq_table()
359 	idx = dmc->opp_count - 1;  in exynos5_init_freq_table()
360 	for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {  in exynos5_init_freq_table()
363 		opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);  in exynos5_init_freq_table()
367 		dmc->opp[idx - i].freq_hz = freq;  in exynos5_init_freq_table()
368 		dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);  in exynos5_init_freq_table()
384 static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)  in exynos5_set_bypass_dram_timings()  argument
387 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);  in exynos5_set_bypass_dram_timings()
389 	writel(dmc->bypass_timing_row,  in exynos5_set_bypass_dram_timings()
390 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);  in exynos5_set_bypass_dram_timings()
391 	writel(dmc->bypass_timing_row,  in exynos5_set_bypass_dram_timings()
392 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);  in exynos5_set_bypass_dram_timings()
393 	writel(dmc->bypass_timing_data,  in exynos5_set_bypass_dram_timings()
394 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);  in exynos5_set_bypass_dram_timings()
395 	writel(dmc->bypass_timing_data,  in exynos5_set_bypass_dram_timings()
396 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);  in exynos5_set_bypass_dram_timings()
397 	writel(dmc->bypass_timing_power,  in exynos5_set_bypass_dram_timings()
398 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);  in exynos5_set_bypass_dram_timings()
399 	writel(dmc->bypass_timing_power,  in exynos5_set_bypass_dram_timings()
400 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);  in exynos5_set_bypass_dram_timings()
414 static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,  in exynos5_dram_change_timings()  argument
419 	for (idx = dmc->opp_count - 1; idx >= 0; idx--)  in exynos5_dram_change_timings()
420 		if (dmc->opp[idx].freq_hz <= target_rate)  in exynos5_dram_change_timings()
427 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);  in exynos5_dram_change_timings()
429 	writel(dmc->timing_row[idx],  in exynos5_dram_change_timings()
430 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);  in exynos5_dram_change_timings()
431 	writel(dmc->timing_row[idx],  in exynos5_dram_change_timings()
432 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);  in exynos5_dram_change_timings()
433 	writel(dmc->timing_data[idx],  in exynos5_dram_change_timings()
434 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);  in exynos5_dram_change_timings()
435 	writel(dmc->timing_data[idx],  in exynos5_dram_change_timings()
436 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);  in exynos5_dram_change_timings()
437 	writel(dmc->timing_power[idx],  in exynos5_dram_change_timings()
438 	       dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);  in exynos5_dram_change_timings()
439 	writel(dmc->timing_power[idx],  in exynos5_dram_change_timings()
440 	       dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);  in exynos5_dram_change_timings()
455 static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,  in exynos5_dmc_align_target_voltage()  argument
460 	if (dmc->curr_volt <= target_volt)  in exynos5_dmc_align_target_voltage()
463 	ret = regulator_set_voltage(dmc->vdd_mif, target_volt,  in exynos5_dmc_align_target_voltage()
466 		dmc->curr_volt = target_volt;  in exynos5_dmc_align_target_voltage()
481 static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,  in exynos5_dmc_align_bypass_voltage()  argument
486 	if (dmc->curr_volt >= target_volt)  in exynos5_dmc_align_bypass_voltage()
489 	ret = regulator_set_voltage(dmc->vdd_mif, target_volt,  in exynos5_dmc_align_bypass_voltage()
492 		dmc->curr_volt = target_volt;  in exynos5_dmc_align_bypass_voltage()
504 static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,  in exynos5_dmc_align_bypass_dram_timings()  argument
507 	int idx = find_target_freq_idx(dmc, target_rate);  in exynos5_dmc_align_bypass_dram_timings()
512 	exynos5_set_bypass_dram_timings(dmc);  in exynos5_dmc_align_bypass_dram_timings()
529 exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,  in exynos5_dmc_switch_to_bypass_configuration()  argument
540 	ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);  in exynos5_dmc_switch_to_bypass_configuration()
547 	ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);  in exynos5_dmc_switch_to_bypass_configuration()
554 	ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);  in exynos5_dmc_switch_to_bypass_configuration()
583 exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,  in exynos5_dmc_change_freq_and_volt()  argument
589 	ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,  in exynos5_dmc_change_freq_and_volt()
598 	clk_prepare_enable(dmc->fout_spll);  in exynos5_dmc_change_freq_and_volt()
599 	clk_prepare_enable(dmc->mout_spll);  in exynos5_dmc_change_freq_and_volt()
600 	clk_prepare_enable(dmc->mout_mx_mspll_ccore);  in exynos5_dmc_change_freq_and_volt()
602 	ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);  in exynos5_dmc_change_freq_and_volt()
611 	exynos5_dram_change_timings(dmc, target_rate);  in exynos5_dmc_change_freq_and_volt()
613 	clk_set_rate(dmc->fout_bpll, target_rate);  in exynos5_dmc_change_freq_and_volt()
615 	ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);  in exynos5_dmc_change_freq_and_volt()
619 	ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);  in exynos5_dmc_change_freq_and_volt()
627 	ret = exynos5_dmc_align_target_voltage(dmc, target_volt);  in exynos5_dmc_change_freq_and_volt()
630 	clk_disable_unprepare(dmc->mout_mx_mspll_ccore);  in exynos5_dmc_change_freq_and_volt()
631 	clk_disable_unprepare(dmc->mout_spll);  in exynos5_dmc_change_freq_and_volt()
632 	clk_disable_unprepare(dmc->fout_spll);  in exynos5_dmc_change_freq_and_volt()
652 static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,  in exynos5_dmc_get_volt_freq()  argument
659 	opp = devfreq_recommended_opp(dmc->dev, freq, flags);  in exynos5_dmc_get_volt_freq()
685 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);  in exynos5_dmc_target()  local
690 	ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,  in exynos5_dmc_target()
696 	if (target_rate == dmc->curr_rate)  in exynos5_dmc_target()
699 	mutex_lock(&dmc->lock);  in exynos5_dmc_target()
701 	ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);  in exynos5_dmc_target()
704 		mutex_unlock(&dmc->lock);  in exynos5_dmc_target()
708 	dmc->curr_rate = target_rate;  in exynos5_dmc_target()
710 	mutex_unlock(&dmc->lock);  in exynos5_dmc_target()
724 static int exynos5_counters_get(struct exynos5_dmc *dmc,  in exynos5_counters_get()  argument
735 	for (i = 0; i < dmc->num_counters; i++) {  in exynos5_counters_get()
736 		if (!dmc->counter[i])  in exynos5_counters_get()
739 		ret = devfreq_event_get_event(dmc->counter[i], &event);  in exynos5_counters_get()
762 static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,  in exynos5_dmc_start_perf_events()  argument
766 	writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);  in exynos5_dmc_start_perf_events()
767 	writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);  in exynos5_dmc_start_perf_events()
770 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);  in exynos5_dmc_start_perf_events()
771 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);  in exynos5_dmc_start_perf_events()
774 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);  in exynos5_dmc_start_perf_events()
775 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);  in exynos5_dmc_start_perf_events()
778 	writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);  in exynos5_dmc_start_perf_events()
779 	writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);  in exynos5_dmc_start_perf_events()
785 	writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);  in exynos5_dmc_start_perf_events()
786 	writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);  in exynos5_dmc_start_perf_events()
789 	writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);  in exynos5_dmc_start_perf_events()
790 	writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);  in exynos5_dmc_start_perf_events()
802 static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)  in exynos5_dmc_perf_events_calc()  argument
819 		dmc->load = 70;  in exynos5_dmc_perf_events_calc()
820 		dmc->total = 100;  in exynos5_dmc_perf_events_calc()
826 		dmc->load = 35;  in exynos5_dmc_perf_events_calc()
827 		dmc->total = 100;  in exynos5_dmc_perf_events_calc()
830 	dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);  in exynos5_dmc_perf_events_calc()
840 static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)  in exynos5_dmc_perf_events_check()  argument
848 	writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);  in exynos5_dmc_perf_events_check()
849 	writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);  in exynos5_dmc_perf_events_check()
852 	val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);  in exynos5_dmc_perf_events_check()
854 		diff_ts = ts - dmc->last_overflow_ts[0];  in exynos5_dmc_perf_events_check()
855 		dmc->last_overflow_ts[0] = ts;  in exynos5_dmc_perf_events_check()
856 		dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n",  val);  in exynos5_dmc_perf_events_check()
858 		val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);  in exynos5_dmc_perf_events_check()
859 		diff_ts = ts - dmc->last_overflow_ts[1];  in exynos5_dmc_perf_events_check()
860 		dmc->last_overflow_ts[1] = ts;  in exynos5_dmc_perf_events_check()
861 		dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n",  val);  in exynos5_dmc_perf_events_check()
864 	exynos5_dmc_perf_events_calc(dmc, diff_ts);  in exynos5_dmc_perf_events_check()
866 	exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);  in exynos5_dmc_perf_events_check()
875 static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)  in exynos5_dmc_enable_perf_events()  argument
880 	writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);  in exynos5_dmc_enable_perf_events()
881 	writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);  in exynos5_dmc_enable_perf_events()
884 	writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);  in exynos5_dmc_enable_perf_events()
885 	writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);  in exynos5_dmc_enable_perf_events()
888 	dmc->last_overflow_ts[0] = ts;  in exynos5_dmc_enable_perf_events()
889 	dmc->last_overflow_ts[1] = ts;  in exynos5_dmc_enable_perf_events()
892 	dmc->load = 99;  in exynos5_dmc_enable_perf_events()
893 	dmc->total = 100;  in exynos5_dmc_enable_perf_events()
902 static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)  in exynos5_dmc_disable_perf_events()  argument
905 	writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);  in exynos5_dmc_disable_perf_events()
906 	writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);  in exynos5_dmc_disable_perf_events()
909 	writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);  in exynos5_dmc_disable_perf_events()
910 	writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);  in exynos5_dmc_disable_perf_events()
913 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);  in exynos5_dmc_disable_perf_events()
914 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);  in exynos5_dmc_disable_perf_events()
917 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);  in exynos5_dmc_disable_perf_events()
918 	writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);  in exynos5_dmc_disable_perf_events()
933 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);  in exynos5_dmc_get_status()  local
937 	if (dmc->in_irq_mode) {  in exynos5_dmc_get_status()
938 		mutex_lock(&dmc->lock);  in exynos5_dmc_get_status()
939 		stat->current_frequency = dmc->curr_rate;  in exynos5_dmc_get_status()
940 		mutex_unlock(&dmc->lock);  in exynos5_dmc_get_status()
942 		stat->busy_time = dmc->load;  in exynos5_dmc_get_status()
943 		stat->total_time = dmc->total;  in exynos5_dmc_get_status()
945 		ret = exynos5_counters_get(dmc, &load, &total);  in exynos5_dmc_get_status()
953 		ret = exynos5_counters_set_event(dmc);  in exynos5_dmc_get_status()
974 	struct exynos5_dmc *dmc = dev_get_drvdata(dev);  in exynos5_dmc_get_cur_freq()  local
976 	mutex_lock(&dmc->lock);  in exynos5_dmc_get_cur_freq()
977 	*freq = dmc->curr_rate;  in exynos5_dmc_get_cur_freq()
978 	mutex_unlock(&dmc->lock);  in exynos5_dmc_get_cur_freq()
1008 exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,  in exynos5_dmc_align_init_freq()  argument
1014 	idx = find_target_freq_idx(dmc, bootloader_init_freq);  in exynos5_dmc_align_init_freq()
1016 		aligned_freq = dmc->opp[idx].freq_hz;  in exynos5_dmc_align_init_freq()
1018 		aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;  in exynos5_dmc_align_init_freq()
1036 static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,  in create_timings_aligned()  argument
1050 	val = dmc->timings->tRFC / clk_period_ps;  in create_timings_aligned()
1051 	val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1052 	val = max(val, dmc->min_tck->tRFC);  in create_timings_aligned()
1056 	val = dmc->timings->tRRD / clk_period_ps;  in create_timings_aligned()
1057 	val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1058 	val = max(val, dmc->min_tck->tRRD);  in create_timings_aligned()
1062 	val = dmc->timings->tRPab / clk_period_ps;  in create_timings_aligned()
1063 	val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1064 	val = max(val, dmc->min_tck->tRPab);  in create_timings_aligned()
1068 	val = dmc->timings->tRCD / clk_period_ps;  in create_timings_aligned()
1069 	val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1070 	val = max(val, dmc->min_tck->tRCD);  in create_timings_aligned()
1074 	val = dmc->timings->tRC / clk_period_ps;  in create_timings_aligned()
1075 	val += dmc->timings->tRC % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1076 	val = max(val, dmc->min_tck->tRC);  in create_timings_aligned()
1080 	val = dmc->timings->tRAS / clk_period_ps;  in create_timings_aligned()
1081 	val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1082 	val = max(val, dmc->min_tck->tRAS);  in create_timings_aligned()
1087 	val = dmc->timings->tWTR / clk_period_ps;  in create_timings_aligned()
1088 	val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1089 	val = max(val, dmc->min_tck->tWTR);  in create_timings_aligned()
1093 	val = dmc->timings->tWR / clk_period_ps;  in create_timings_aligned()
1094 	val += dmc->timings->tWR % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1095 	val = max(val, dmc->min_tck->tWR);  in create_timings_aligned()
1099 	val = dmc->timings->tRTP / clk_period_ps;  in create_timings_aligned()
1100 	val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1101 	val = max(val, dmc->min_tck->tRTP);  in create_timings_aligned()
1105 	val = dmc->timings->tW2W_C2C / clk_period_ps;  in create_timings_aligned()
1106 	val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1107 	val = max(val, dmc->min_tck->tW2W_C2C);  in create_timings_aligned()
1111 	val = dmc->timings->tR2R_C2C / clk_period_ps;  in create_timings_aligned()
1112 	val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1113 	val = max(val, dmc->min_tck->tR2R_C2C);  in create_timings_aligned()
1117 	val = dmc->timings->tWL / clk_period_ps;  in create_timings_aligned()
1118 	val += dmc->timings->tWL % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1119 	val = max(val, dmc->min_tck->tWL);  in create_timings_aligned()
1123 	val = dmc->timings->tDQSCK / clk_period_ps;  in create_timings_aligned()
1124 	val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1125 	val = max(val, dmc->min_tck->tDQSCK);  in create_timings_aligned()
1129 	val = dmc->timings->tRL / clk_period_ps;  in create_timings_aligned()
1130 	val += dmc->timings->tRL % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1131 	val = max(val, dmc->min_tck->tRL);  in create_timings_aligned()
1136 	val = dmc->timings->tFAW / clk_period_ps;  in create_timings_aligned()
1137 	val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1138 	val = max(val, dmc->min_tck->tFAW);  in create_timings_aligned()
1142 	val = dmc->timings->tXSR / clk_period_ps;  in create_timings_aligned()
1143 	val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1144 	val = max(val, dmc->min_tck->tXSR);  in create_timings_aligned()
1148 	val = dmc->timings->tXP / clk_period_ps;  in create_timings_aligned()
1149 	val += dmc->timings->tXP % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1150 	val = max(val, dmc->min_tck->tXP);  in create_timings_aligned()
1154 	val = dmc->timings->tCKE / clk_period_ps;  in create_timings_aligned()
1155 	val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1156 	val = max(val, dmc->min_tck->tCKE);  in create_timings_aligned()
1160 	val = dmc->timings->tMRD / clk_period_ps;  in create_timings_aligned()
1161 	val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;  in create_timings_aligned()
1162 	val = max(val, dmc->min_tck->tMRD);  in create_timings_aligned()
1175 static int of_get_dram_timings(struct exynos5_dmc *dmc)  in of_get_dram_timings()  argument
1182 	np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);  in of_get_dram_timings()
1184 		dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");  in of_get_dram_timings()
1188 	dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,  in of_get_dram_timings()
1190 	if (!dmc->timing_row) {  in of_get_dram_timings()
1195 	dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,  in of_get_dram_timings()
1197 	if (!dmc->timing_data) {  in of_get_dram_timings()
1202 	dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,  in of_get_dram_timings()
1204 	if (!dmc->timing_power) {  in of_get_dram_timings()
1209 	dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,  in of_get_dram_timings()
1211 						 &dmc->timings_arr_size);  in of_get_dram_timings()
1212 	if (!dmc->timings) {  in of_get_dram_timings()
1213 		dev_warn(dmc->dev, "could not get timings from DT\n");  in of_get_dram_timings()
1218 	dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);  in of_get_dram_timings()
1219 	if (!dmc->min_tck) {  in of_get_dram_timings()
1220 		dev_warn(dmc->dev, "could not get tck from DT\n");  in of_get_dram_timings()
1226 	for (idx = 0; idx < dmc->opp_count; idx++) {  in of_get_dram_timings()
1227 		freq_mhz = dmc->opp[idx].freq_hz / 1000000;  in of_get_dram_timings()
1230 		ret = create_timings_aligned(dmc, &dmc->timing_row[idx],  in of_get_dram_timings()
1231 					     &dmc->timing_data[idx],  in of_get_dram_timings()
1232 					     &dmc->timing_power[idx],  in of_get_dram_timings()
1238 	dmc->bypass_timing_row = dmc->timing_row[idx - 1];  in of_get_dram_timings()
1239 	dmc->bypass_timing_data = dmc->timing_data[idx - 1];  in of_get_dram_timings()
1240 	dmc->bypass_timing_power = dmc->timing_power[idx - 1];  in of_get_dram_timings()
1254 static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)  in exynos5_dmc_init_clks()  argument
1261 	dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");  in exynos5_dmc_init_clks()
1262 	if (IS_ERR(dmc->fout_spll))  in exynos5_dmc_init_clks()
1263 		return PTR_ERR(dmc->fout_spll);  in exynos5_dmc_init_clks()
1265 	dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");  in exynos5_dmc_init_clks()
1266 	if (IS_ERR(dmc->fout_bpll))  in exynos5_dmc_init_clks()
1267 		return PTR_ERR(dmc->fout_bpll);  in exynos5_dmc_init_clks()
1269 	dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");  in exynos5_dmc_init_clks()
1270 	if (IS_ERR(dmc->mout_mclk_cdrex))  in exynos5_dmc_init_clks()
1271 		return PTR_ERR(dmc->mout_mclk_cdrex);  in exynos5_dmc_init_clks()
1273 	dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");  in exynos5_dmc_init_clks()
1274 	if (IS_ERR(dmc->mout_bpll))  in exynos5_dmc_init_clks()
1275 		return PTR_ERR(dmc->mout_bpll);  in exynos5_dmc_init_clks()
1277 	dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,  in exynos5_dmc_init_clks()
1279 	if (IS_ERR(dmc->mout_mx_mspll_ccore))  in exynos5_dmc_init_clks()
1280 		return PTR_ERR(dmc->mout_mx_mspll_ccore);  in exynos5_dmc_init_clks()
1282 	dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");  in exynos5_dmc_init_clks()
1283 	if (IS_ERR(dmc->mout_spll)) {  in exynos5_dmc_init_clks()
1284 		dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");  in exynos5_dmc_init_clks()
1285 		if (IS_ERR(dmc->mout_spll))  in exynos5_dmc_init_clks()
1286 			return PTR_ERR(dmc->mout_spll);  in exynos5_dmc_init_clks()
1292 	dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);  in exynos5_dmc_init_clks()
1293 	dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);  in exynos5_dmc_init_clks()
1294 	exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;  in exynos5_dmc_init_clks()
1296 	ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,  in exynos5_dmc_init_clks()
1301 	dmc->curr_volt = target_volt;  in exynos5_dmc_init_clks()
1303 	ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);  in exynos5_dmc_init_clks()
1307 	clk_prepare_enable(dmc->fout_bpll);  in exynos5_dmc_init_clks()
1308 	clk_prepare_enable(dmc->mout_bpll);  in exynos5_dmc_init_clks()
1314 	regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);  in exynos5_dmc_init_clks()
1316 	regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);  in exynos5_dmc_init_clks()
1330 static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)  in exynos5_performance_counters_init()  argument
1334 	dmc->num_counters = devfreq_event_get_edev_count(dmc->dev,  in exynos5_performance_counters_init()
1336 	if (dmc->num_counters < 0) {  in exynos5_performance_counters_init()
1337 		dev_err(dmc->dev, "could not get devfreq-event counters\n");  in exynos5_performance_counters_init()
1338 		return dmc->num_counters;  in exynos5_performance_counters_init()
1341 	dmc->counter = devm_kcalloc(dmc->dev, dmc->num_counters,  in exynos5_performance_counters_init()
1342 				    sizeof(*dmc->counter), GFP_KERNEL);  in exynos5_performance_counters_init()
1343 	if (!dmc->counter)  in exynos5_performance_counters_init()
1346 	for (i = 0; i < dmc->num_counters; i++) {  in exynos5_performance_counters_init()
1347 		dmc->counter[i] =  in exynos5_performance_counters_init()
1348 			devfreq_event_get_edev_by_phandle(dmc->dev,  in exynos5_performance_counters_init()
1350 		if (IS_ERR_OR_NULL(dmc->counter[i]))  in exynos5_performance_counters_init()
1354 	ret = exynos5_counters_enable_edev(dmc);  in exynos5_performance_counters_init()
1356 		dev_err(dmc->dev, "could not enable event counter\n");  in exynos5_performance_counters_init()
1360 	ret = exynos5_counters_set_event(dmc);  in exynos5_performance_counters_init()
1362 		exynos5_counters_disable_edev(dmc);  in exynos5_performance_counters_init()
1363 		dev_err(dmc->dev, "could not set event counter\n");  in exynos5_performance_counters_init()
1379 static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)  in exynos5_dmc_set_pause_on_switching()  argument
1384 	ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);  in exynos5_dmc_set_pause_on_switching()
1389 	regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);  in exynos5_dmc_set_pause_on_switching()
1397 	struct exynos5_dmc *dmc = priv;  in dmc_irq_thread()  local
1399 	mutex_lock(&dmc->df->lock);  in dmc_irq_thread()
1400 	exynos5_dmc_perf_events_check(dmc);  in dmc_irq_thread()
1401 	res = update_devfreq(dmc->df);  in dmc_irq_thread()
1402 	mutex_unlock(&dmc->df->lock);  in dmc_irq_thread()
1405 		dev_warn(dmc->dev, "devfreq failed with %d\n", res);  in dmc_irq_thread()
1425 	struct exynos5_dmc *dmc;  in exynos5_dmc_probe()  local
1428 	dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);  in exynos5_dmc_probe()
1429 	if (!dmc)  in exynos5_dmc_probe()
1432 	mutex_init(&dmc->lock);  in exynos5_dmc_probe()
1434 	dmc->dev = dev;  in exynos5_dmc_probe()
1435 	platform_set_drvdata(pdev, dmc);  in exynos5_dmc_probe()
1437 	dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);  in exynos5_dmc_probe()
1438 	if (IS_ERR(dmc->base_drexi0))  in exynos5_dmc_probe()
1439 		return PTR_ERR(dmc->base_drexi0);  in exynos5_dmc_probe()
1441 	dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);  in exynos5_dmc_probe()
1442 	if (IS_ERR(dmc->base_drexi1))  in exynos5_dmc_probe()
1443 		return PTR_ERR(dmc->base_drexi1);  in exynos5_dmc_probe()
1445 	dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,  in exynos5_dmc_probe()
1447 	if (IS_ERR(dmc->clk_regmap))  in exynos5_dmc_probe()
1448 		return PTR_ERR(dmc->clk_regmap);  in exynos5_dmc_probe()
1450 	ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);  in exynos5_dmc_probe()
1456 	dmc->vdd_mif = devm_regulator_get(dev, "vdd");  in exynos5_dmc_probe()
1457 	if (IS_ERR(dmc->vdd_mif)) {  in exynos5_dmc_probe()
1458 		ret = PTR_ERR(dmc->vdd_mif);  in exynos5_dmc_probe()
1462 	ret = exynos5_dmc_init_clks(dmc);  in exynos5_dmc_probe()
1466 	ret = of_get_dram_timings(dmc);  in exynos5_dmc_probe()
1472 	ret = exynos5_dmc_set_pause_on_switching(dmc);  in exynos5_dmc_probe()
1484 						dev_name(dev), dmc);  in exynos5_dmc_probe()
1492 						dev_name(dev), dmc);  in exynos5_dmc_probe()
1502 		dmc->gov_data.upthreshold = 55;  in exynos5_dmc_probe()
1503 		dmc->gov_data.downdifferential = 5;  in exynos5_dmc_probe()
1505 		exynos5_dmc_enable_perf_events(dmc);  in exynos5_dmc_probe()
1507 		dmc->in_irq_mode = 1;  in exynos5_dmc_probe()
1509 		ret = exynos5_performance_counters_init(dmc);  in exynos5_dmc_probe()
1519 		dmc->gov_data.upthreshold = 10;  in exynos5_dmc_probe()
1520 		dmc->gov_data.downdifferential = 5;  in exynos5_dmc_probe()
1525 	dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,  in exynos5_dmc_probe()
1527 					  &dmc->gov_data);  in exynos5_dmc_probe()
1529 	if (IS_ERR(dmc->df)) {  in exynos5_dmc_probe()
1530 		ret = PTR_ERR(dmc->df);  in exynos5_dmc_probe()
1534 	if (dmc->in_irq_mode)  in exynos5_dmc_probe()
1535 		exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);  in exynos5_dmc_probe()
1537 	dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);  in exynos5_dmc_probe()
1542 	if (dmc->in_irq_mode)  in exynos5_dmc_probe()
1543 		exynos5_dmc_disable_perf_events(dmc);  in exynos5_dmc_probe()
1545 		exynos5_counters_disable_edev(dmc);  in exynos5_dmc_probe()
1547 	clk_disable_unprepare(dmc->mout_bpll);  in exynos5_dmc_probe()
1548 	clk_disable_unprepare(dmc->fout_bpll);  in exynos5_dmc_probe()
1563 	struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);  in exynos5_dmc_remove()  local
1565 	if (dmc->in_irq_mode)  in exynos5_dmc_remove()
1566 		exynos5_dmc_disable_perf_events(dmc);  in exynos5_dmc_remove()
1568 		exynos5_counters_disable_edev(dmc);  in exynos5_dmc_remove()
1570 	clk_disable_unprepare(dmc->mout_bpll);  in exynos5_dmc_remove()
1571 	clk_disable_unprepare(dmc->fout_bpll);  in exynos5_dmc_remove()