1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Linaro Ltd.
4 *
5 * Author: Stanimir Varbanov <stanimir.varbanov@linaro.org>
6 */
7 #include <linux/clk.h>
8 #include <linux/interconnect.h>
9 #include <linux/iopoll.h>
10 #include <linux/kernel.h>
11 #include <linux/pm_domain.h>
12 #include <linux/pm_opp.h>
13 #include <linux/pm_runtime.h>
14 #include <linux/reset.h>
15 #include <linux/types.h>
16 #include <media/v4l2-mem2mem.h>
17
18 #include "core.h"
19 #include "hfi_parser.h"
20 #include "hfi_venus_io.h"
21 #include "pm_helpers.h"
22 #include "hfi_platform.h"
23
24 static bool legacy_binding;
25
core_clks_get(struct venus_core * core)26 static int core_clks_get(struct venus_core *core)
27 {
28 const struct venus_resources *res = core->res;
29 struct device *dev = core->dev;
30 unsigned int i;
31
32 for (i = 0; i < res->clks_num; i++) {
33 core->clks[i] = devm_clk_get(dev, res->clks[i]);
34 if (IS_ERR(core->clks[i]))
35 return PTR_ERR(core->clks[i]);
36 }
37
38 return 0;
39 }
40
core_clks_enable(struct venus_core * core)41 static int core_clks_enable(struct venus_core *core)
42 {
43 const struct venus_resources *res = core->res;
44 const struct freq_tbl *freq_tbl = core->res->freq_tbl;
45 unsigned int freq_tbl_size = core->res->freq_tbl_size;
46 unsigned long freq;
47 unsigned int i;
48 int ret;
49
50 if (!freq_tbl)
51 return -EINVAL;
52
53 freq = freq_tbl[freq_tbl_size - 1].freq;
54
55 for (i = 0; i < res->clks_num; i++) {
56 if (IS_V6(core)) {
57 ret = clk_set_rate(core->clks[i], freq);
58 if (ret)
59 goto err;
60 }
61
62 ret = clk_prepare_enable(core->clks[i]);
63 if (ret)
64 goto err;
65 }
66
67 return 0;
68 err:
69 while (i--)
70 clk_disable_unprepare(core->clks[i]);
71
72 return ret;
73 }
74
core_clks_disable(struct venus_core * core)75 static void core_clks_disable(struct venus_core *core)
76 {
77 const struct venus_resources *res = core->res;
78 unsigned int i = res->clks_num;
79
80 while (i--)
81 clk_disable_unprepare(core->clks[i]);
82 }
83
core_clks_set_rate(struct venus_core * core,unsigned long freq)84 static int core_clks_set_rate(struct venus_core *core, unsigned long freq)
85 {
86 int ret;
87
88 ret = dev_pm_opp_set_rate(core->dev, freq);
89 if (ret)
90 return ret;
91
92 ret = clk_set_rate(core->vcodec0_clks[0], freq);
93 if (ret)
94 return ret;
95
96 ret = clk_set_rate(core->vcodec1_clks[0], freq);
97 if (ret)
98 return ret;
99
100 return 0;
101 }
102
vcodec_clks_get(struct venus_core * core,struct device * dev,struct clk ** clks,const char * const * id)103 static int vcodec_clks_get(struct venus_core *core, struct device *dev,
104 struct clk **clks, const char * const *id)
105 {
106 const struct venus_resources *res = core->res;
107 unsigned int i;
108
109 for (i = 0; i < res->vcodec_clks_num; i++) {
110 if (!id[i])
111 continue;
112 clks[i] = devm_clk_get(dev, id[i]);
113 if (IS_ERR(clks[i]))
114 return PTR_ERR(clks[i]);
115 }
116
117 return 0;
118 }
119
vcodec_clks_enable(struct venus_core * core,struct clk ** clks)120 static int vcodec_clks_enable(struct venus_core *core, struct clk **clks)
121 {
122 const struct venus_resources *res = core->res;
123 unsigned int i;
124 int ret;
125
126 for (i = 0; i < res->vcodec_clks_num; i++) {
127 ret = clk_prepare_enable(clks[i]);
128 if (ret)
129 goto err;
130 }
131
132 return 0;
133 err:
134 while (i--)
135 clk_disable_unprepare(clks[i]);
136
137 return ret;
138 }
139
vcodec_clks_disable(struct venus_core * core,struct clk ** clks)140 static void vcodec_clks_disable(struct venus_core *core, struct clk **clks)
141 {
142 const struct venus_resources *res = core->res;
143 unsigned int i = res->vcodec_clks_num;
144
145 while (i--)
146 clk_disable_unprepare(clks[i]);
147 }
148
load_per_instance(struct venus_inst * inst)149 static u32 load_per_instance(struct venus_inst *inst)
150 {
151 u32 mbs;
152
153 if (!inst || !(inst->state >= INST_INIT && inst->state < INST_STOP))
154 return 0;
155
156 mbs = (ALIGN(inst->width, 16) / 16) * (ALIGN(inst->height, 16) / 16);
157
158 return mbs * inst->fps;
159 }
160
load_per_type(struct venus_core * core,u32 session_type)161 static u32 load_per_type(struct venus_core *core, u32 session_type)
162 {
163 struct venus_inst *inst = NULL;
164 u32 mbs_per_sec = 0;
165
166 mutex_lock(&core->lock);
167 list_for_each_entry(inst, &core->instances, list) {
168 if (inst->session_type != session_type)
169 continue;
170
171 mbs_per_sec += load_per_instance(inst);
172 }
173 mutex_unlock(&core->lock);
174
175 return mbs_per_sec;
176 }
177
mbs_to_bw(struct venus_inst * inst,u32 mbs,u32 * avg,u32 * peak)178 static void mbs_to_bw(struct venus_inst *inst, u32 mbs, u32 *avg, u32 *peak)
179 {
180 const struct venus_resources *res = inst->core->res;
181 const struct bw_tbl *bw_tbl;
182 unsigned int num_rows, i;
183
184 *avg = 0;
185 *peak = 0;
186
187 if (mbs == 0)
188 return;
189
190 if (inst->session_type == VIDC_SESSION_TYPE_ENC) {
191 num_rows = res->bw_tbl_enc_size;
192 bw_tbl = res->bw_tbl_enc;
193 } else if (inst->session_type == VIDC_SESSION_TYPE_DEC) {
194 num_rows = res->bw_tbl_dec_size;
195 bw_tbl = res->bw_tbl_dec;
196 } else {
197 return;
198 }
199
200 if (!bw_tbl || num_rows == 0)
201 return;
202
203 for (i = 0; i < num_rows; i++) {
204 if (i != 0 && mbs > bw_tbl[i].mbs_per_sec)
205 break;
206
207 if (inst->dpb_fmt & HFI_COLOR_FORMAT_10_BIT_BASE) {
208 *avg = bw_tbl[i].avg_10bit;
209 *peak = bw_tbl[i].peak_10bit;
210 } else {
211 *avg = bw_tbl[i].avg;
212 *peak = bw_tbl[i].peak;
213 }
214 }
215 }
216
load_scale_bw(struct venus_core * core)217 static int load_scale_bw(struct venus_core *core)
218 {
219 struct venus_inst *inst = NULL;
220 u32 mbs_per_sec, avg, peak, total_avg = 0, total_peak = 0;
221
222 mutex_lock(&core->lock);
223 list_for_each_entry(inst, &core->instances, list) {
224 mbs_per_sec = load_per_instance(inst);
225 mbs_to_bw(inst, mbs_per_sec, &avg, &peak);
226 total_avg += avg;
227 total_peak += peak;
228 }
229 mutex_unlock(&core->lock);
230
231 /*
232 * keep minimum bandwidth vote for "video-mem" path,
233 * so that clks can be disabled during vdec_session_release().
234 * Actual bandwidth drop will be done during device supend
235 * so that device can power down without any warnings.
236 */
237
238 if (!total_avg && !total_peak)
239 total_avg = kbps_to_icc(1000);
240
241 dev_dbg(core->dev, VDBGL "total: avg_bw: %u, peak_bw: %u\n",
242 total_avg, total_peak);
243
244 return icc_set_bw(core->video_path, total_avg, total_peak);
245 }
246
load_scale_v1(struct venus_inst * inst)247 static int load_scale_v1(struct venus_inst *inst)
248 {
249 struct venus_core *core = inst->core;
250 const struct freq_tbl *table = core->res->freq_tbl;
251 unsigned int num_rows = core->res->freq_tbl_size;
252 unsigned long freq = table[0].freq;
253 struct device *dev = core->dev;
254 u32 mbs_per_sec;
255 unsigned int i;
256 int ret;
257
258 mbs_per_sec = load_per_type(core, VIDC_SESSION_TYPE_ENC) +
259 load_per_type(core, VIDC_SESSION_TYPE_DEC);
260
261 if (mbs_per_sec > core->res->max_load)
262 dev_warn(dev, "HW is overloaded, needed: %d max: %d\n",
263 mbs_per_sec, core->res->max_load);
264
265 if (!mbs_per_sec && num_rows > 1) {
266 freq = table[num_rows - 1].freq;
267 goto set_freq;
268 }
269
270 for (i = 0; i < num_rows; i++) {
271 if (mbs_per_sec > table[i].load)
272 break;
273 freq = table[i].freq;
274 }
275
276 set_freq:
277
278 ret = core_clks_set_rate(core, freq);
279 if (ret) {
280 dev_err(dev, "failed to set clock rate %lu (%d)\n",
281 freq, ret);
282 return ret;
283 }
284
285 ret = load_scale_bw(core);
286 if (ret) {
287 dev_err(dev, "failed to set bandwidth (%d)\n",
288 ret);
289 return ret;
290 }
291
292 return 0;
293 }
294
core_get_v1(struct venus_core * core)295 static int core_get_v1(struct venus_core *core)
296 {
297 int ret;
298
299 ret = core_clks_get(core);
300 if (ret)
301 return ret;
302
303 ret = devm_pm_opp_set_clkname(core->dev, "core");
304 if (ret)
305 return ret;
306
307 return 0;
308 }
309
core_put_v1(struct venus_core * core)310 static void core_put_v1(struct venus_core *core)
311 {
312 }
313
core_power_v1(struct venus_core * core,int on)314 static int core_power_v1(struct venus_core *core, int on)
315 {
316 int ret = 0;
317
318 if (on == POWER_ON)
319 ret = core_clks_enable(core);
320 else
321 core_clks_disable(core);
322
323 return ret;
324 }
325
326 static const struct venus_pm_ops pm_ops_v1 = {
327 .core_get = core_get_v1,
328 .core_put = core_put_v1,
329 .core_power = core_power_v1,
330 .load_scale = load_scale_v1,
331 };
332
333 static void
vcodec_control_v3(struct venus_core * core,u32 session_type,bool enable)334 vcodec_control_v3(struct venus_core *core, u32 session_type, bool enable)
335 {
336 void __iomem *ctrl;
337
338 if (session_type == VIDC_SESSION_TYPE_DEC)
339 ctrl = core->wrapper_base + WRAPPER_VDEC_VCODEC_POWER_CONTROL;
340 else
341 ctrl = core->wrapper_base + WRAPPER_VENC_VCODEC_POWER_CONTROL;
342
343 if (enable)
344 writel(0, ctrl);
345 else
346 writel(1, ctrl);
347 }
348
vdec_get_v3(struct device * dev)349 static int vdec_get_v3(struct device *dev)
350 {
351 struct venus_core *core = dev_get_drvdata(dev);
352
353 return vcodec_clks_get(core, dev, core->vcodec0_clks,
354 core->res->vcodec0_clks);
355 }
356
vdec_power_v3(struct device * dev,int on)357 static int vdec_power_v3(struct device *dev, int on)
358 {
359 struct venus_core *core = dev_get_drvdata(dev);
360 int ret = 0;
361
362 vcodec_control_v3(core, VIDC_SESSION_TYPE_DEC, true);
363
364 if (on == POWER_ON)
365 ret = vcodec_clks_enable(core, core->vcodec0_clks);
366 else
367 vcodec_clks_disable(core, core->vcodec0_clks);
368
369 vcodec_control_v3(core, VIDC_SESSION_TYPE_DEC, false);
370
371 return ret;
372 }
373
venc_get_v3(struct device * dev)374 static int venc_get_v3(struct device *dev)
375 {
376 struct venus_core *core = dev_get_drvdata(dev);
377
378 return vcodec_clks_get(core, dev, core->vcodec1_clks,
379 core->res->vcodec1_clks);
380 }
381
venc_power_v3(struct device * dev,int on)382 static int venc_power_v3(struct device *dev, int on)
383 {
384 struct venus_core *core = dev_get_drvdata(dev);
385 int ret = 0;
386
387 vcodec_control_v3(core, VIDC_SESSION_TYPE_ENC, true);
388
389 if (on == POWER_ON)
390 ret = vcodec_clks_enable(core, core->vcodec1_clks);
391 else
392 vcodec_clks_disable(core, core->vcodec1_clks);
393
394 vcodec_control_v3(core, VIDC_SESSION_TYPE_ENC, false);
395
396 return ret;
397 }
398
399 static const struct venus_pm_ops pm_ops_v3 = {
400 .core_get = core_get_v1,
401 .core_put = core_put_v1,
402 .core_power = core_power_v1,
403 .vdec_get = vdec_get_v3,
404 .vdec_power = vdec_power_v3,
405 .venc_get = venc_get_v3,
406 .venc_power = venc_power_v3,
407 .load_scale = load_scale_v1,
408 };
409
vcodec_control_v4(struct venus_core * core,u32 coreid,bool enable)410 static int vcodec_control_v4(struct venus_core *core, u32 coreid, bool enable)
411 {
412 void __iomem *ctrl, *stat;
413 u32 val;
414 int ret;
415
416 if (IS_V6(core)) {
417 ctrl = core->wrapper_base + WRAPPER_CORE_POWER_CONTROL_V6;
418 stat = core->wrapper_base + WRAPPER_CORE_POWER_STATUS_V6;
419 } else if (coreid == VIDC_CORE_ID_1) {
420 ctrl = core->wrapper_base + WRAPPER_VCODEC0_MMCC_POWER_CONTROL;
421 stat = core->wrapper_base + WRAPPER_VCODEC0_MMCC_POWER_STATUS;
422 } else {
423 ctrl = core->wrapper_base + WRAPPER_VCODEC1_MMCC_POWER_CONTROL;
424 stat = core->wrapper_base + WRAPPER_VCODEC1_MMCC_POWER_STATUS;
425 }
426
427 if (enable) {
428 writel(0, ctrl);
429
430 ret = readl_poll_timeout(stat, val, val & BIT(1), 1, 100);
431 if (ret)
432 return ret;
433 } else {
434 writel(1, ctrl);
435
436 ret = readl_poll_timeout(stat, val, !(val & BIT(1)), 1, 100);
437 if (ret)
438 return ret;
439 }
440
441 return 0;
442 }
443
poweroff_coreid(struct venus_core * core,unsigned int coreid_mask)444 static int poweroff_coreid(struct venus_core *core, unsigned int coreid_mask)
445 {
446 int ret;
447
448 if (coreid_mask & VIDC_CORE_ID_1) {
449 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, true);
450 if (ret)
451 return ret;
452
453 vcodec_clks_disable(core, core->vcodec0_clks);
454
455 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, false);
456 if (ret)
457 return ret;
458
459 ret = pm_runtime_put_sync(core->pmdomains[1]);
460 if (ret < 0)
461 return ret;
462 }
463
464 if (coreid_mask & VIDC_CORE_ID_2) {
465 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, true);
466 if (ret)
467 return ret;
468
469 vcodec_clks_disable(core, core->vcodec1_clks);
470
471 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, false);
472 if (ret)
473 return ret;
474
475 ret = pm_runtime_put_sync(core->pmdomains[2]);
476 if (ret < 0)
477 return ret;
478 }
479
480 return 0;
481 }
482
poweron_coreid(struct venus_core * core,unsigned int coreid_mask)483 static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask)
484 {
485 int ret;
486
487 if (coreid_mask & VIDC_CORE_ID_1) {
488 ret = pm_runtime_get_sync(core->pmdomains[1]);
489 if (ret < 0)
490 return ret;
491
492 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, true);
493 if (ret)
494 return ret;
495
496 ret = vcodec_clks_enable(core, core->vcodec0_clks);
497 if (ret)
498 return ret;
499
500 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, false);
501 if (ret < 0)
502 return ret;
503 }
504
505 if (coreid_mask & VIDC_CORE_ID_2) {
506 ret = pm_runtime_get_sync(core->pmdomains[2]);
507 if (ret < 0)
508 return ret;
509
510 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, true);
511 if (ret)
512 return ret;
513
514 ret = vcodec_clks_enable(core, core->vcodec1_clks);
515 if (ret)
516 return ret;
517
518 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, false);
519 if (ret < 0)
520 return ret;
521 }
522
523 return 0;
524 }
525
power_save_mode_enable(struct venus_inst * inst,bool enable)526 static inline int power_save_mode_enable(struct venus_inst *inst,
527 bool enable)
528 {
529 struct venc_controls *enc_ctr = &inst->controls.enc;
530 const u32 ptype = HFI_PROPERTY_CONFIG_VENC_PERF_MODE;
531 u32 venc_mode;
532 int ret = 0;
533
534 if (inst->session_type != VIDC_SESSION_TYPE_ENC)
535 return 0;
536
537 if (enc_ctr->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CQ)
538 enable = false;
539
540 venc_mode = enable ? HFI_VENC_PERFMODE_POWER_SAVE :
541 HFI_VENC_PERFMODE_MAX_QUALITY;
542
543 ret = hfi_session_set_property(inst, ptype, &venc_mode);
544 if (ret)
545 return ret;
546
547 inst->flags = enable ? inst->flags | VENUS_LOW_POWER :
548 inst->flags & ~VENUS_LOW_POWER;
549
550 return ret;
551 }
552
move_core_to_power_save_mode(struct venus_core * core,u32 core_id)553 static int move_core_to_power_save_mode(struct venus_core *core,
554 u32 core_id)
555 {
556 struct venus_inst *inst = NULL;
557
558 mutex_lock(&core->lock);
559 list_for_each_entry(inst, &core->instances, list) {
560 if (inst->clk_data.core_id == core_id &&
561 inst->session_type == VIDC_SESSION_TYPE_ENC)
562 power_save_mode_enable(inst, true);
563 }
564 mutex_unlock(&core->lock);
565 return 0;
566 }
567
568 static void
min_loaded_core(struct venus_inst * inst,u32 * min_coreid,u32 * min_load,bool low_power)569 min_loaded_core(struct venus_inst *inst, u32 *min_coreid, u32 *min_load, bool low_power)
570 {
571 u32 mbs_per_sec, load, core1_load = 0, core2_load = 0;
572 u32 cores_max = core_num_max(inst);
573 struct venus_core *core = inst->core;
574 struct venus_inst *inst_pos;
575 unsigned long vpp_freq;
576 u32 coreid;
577
578 mutex_lock(&core->lock);
579
580 list_for_each_entry(inst_pos, &core->instances, list) {
581 if (inst_pos == inst)
582 continue;
583
584 if (inst_pos->state != INST_START)
585 continue;
586
587 if (inst->session_type == VIDC_SESSION_TYPE_DEC)
588 vpp_freq = inst_pos->clk_data.vpp_freq;
589 else if (inst->session_type == VIDC_SESSION_TYPE_ENC)
590 vpp_freq = low_power ? inst_pos->clk_data.vpp_freq :
591 inst_pos->clk_data.low_power_freq;
592 else
593 continue;
594
595 coreid = inst_pos->clk_data.core_id;
596
597 mbs_per_sec = load_per_instance(inst_pos);
598 load = mbs_per_sec * vpp_freq;
599
600 if ((coreid & VIDC_CORE_ID_3) == VIDC_CORE_ID_3) {
601 core1_load += load / 2;
602 core2_load += load / 2;
603 } else if (coreid & VIDC_CORE_ID_1) {
604 core1_load += load;
605 } else if (coreid & VIDC_CORE_ID_2) {
606 core2_load += load;
607 }
608 }
609
610 *min_coreid = core1_load <= core2_load ?
611 VIDC_CORE_ID_1 : VIDC_CORE_ID_2;
612 *min_load = min(core1_load, core2_load);
613
614 if (cores_max < VIDC_CORE_ID_2 || core->res->vcodec_num < 2) {
615 *min_coreid = VIDC_CORE_ID_1;
616 *min_load = core1_load;
617 }
618
619 mutex_unlock(&core->lock);
620 }
621
decide_core(struct venus_inst * inst)622 static int decide_core(struct venus_inst *inst)
623 {
624 const u32 ptype = HFI_PROPERTY_CONFIG_VIDEOCORES_USAGE;
625 struct venus_core *core = inst->core;
626 u32 min_coreid, min_load, cur_inst_load;
627 u32 min_lp_coreid, min_lp_load, cur_inst_lp_load;
628 struct hfi_videocores_usage_type cu;
629 unsigned long max_freq;
630 int ret = 0;
631
632 if (legacy_binding) {
633 if (inst->session_type == VIDC_SESSION_TYPE_DEC)
634 cu.video_core_enable_mask = VIDC_CORE_ID_1;
635 else
636 cu.video_core_enable_mask = VIDC_CORE_ID_2;
637
638 goto done;
639 }
640
641 if (inst->clk_data.core_id != VIDC_CORE_ID_DEFAULT)
642 return 0;
643
644 cur_inst_load = load_per_instance(inst);
645 cur_inst_load *= inst->clk_data.vpp_freq;
646 /*TODO : divide this inst->load by work_route */
647
648 cur_inst_lp_load = load_per_instance(inst);
649 cur_inst_lp_load *= inst->clk_data.low_power_freq;
650 /*TODO : divide this inst->load by work_route */
651
652 max_freq = core->res->freq_tbl[0].freq;
653
654 min_loaded_core(inst, &min_coreid, &min_load, false);
655 min_loaded_core(inst, &min_lp_coreid, &min_lp_load, true);
656
657 if (cur_inst_load + min_load <= max_freq) {
658 inst->clk_data.core_id = min_coreid;
659 cu.video_core_enable_mask = min_coreid;
660 } else if (cur_inst_lp_load + min_load <= max_freq) {
661 /* Move current instance to LP and return */
662 inst->clk_data.core_id = min_coreid;
663 cu.video_core_enable_mask = min_coreid;
664 power_save_mode_enable(inst, true);
665 } else if (cur_inst_lp_load + min_lp_load <= max_freq) {
666 /* Move all instances to LP mode and return */
667 inst->clk_data.core_id = min_lp_coreid;
668 cu.video_core_enable_mask = min_lp_coreid;
669 move_core_to_power_save_mode(core, min_lp_coreid);
670 } else {
671 dev_warn(core->dev, "HW can't support this load");
672 return -EINVAL;
673 }
674
675 done:
676 ret = hfi_session_set_property(inst, ptype, &cu);
677 if (ret)
678 return ret;
679
680 return ret;
681 }
682
acquire_core(struct venus_inst * inst)683 static int acquire_core(struct venus_inst *inst)
684 {
685 struct venus_core *core = inst->core;
686 unsigned int coreid_mask = 0;
687
688 if (inst->core_acquired)
689 return 0;
690
691 inst->core_acquired = true;
692
693 if (inst->clk_data.core_id & VIDC_CORE_ID_1) {
694 if (core->core0_usage_count++)
695 return 0;
696
697 coreid_mask = VIDC_CORE_ID_1;
698 }
699
700 if (inst->clk_data.core_id & VIDC_CORE_ID_2) {
701 if (core->core1_usage_count++)
702 return 0;
703
704 coreid_mask |= VIDC_CORE_ID_2;
705 }
706
707 return poweron_coreid(core, coreid_mask);
708 }
709
release_core(struct venus_inst * inst)710 static int release_core(struct venus_inst *inst)
711 {
712 struct venus_core *core = inst->core;
713 unsigned int coreid_mask = 0;
714 int ret;
715
716 if (!inst->core_acquired)
717 return 0;
718
719 if (inst->clk_data.core_id & VIDC_CORE_ID_1) {
720 if (--core->core0_usage_count)
721 goto done;
722
723 coreid_mask = VIDC_CORE_ID_1;
724 }
725
726 if (inst->clk_data.core_id & VIDC_CORE_ID_2) {
727 if (--core->core1_usage_count)
728 goto done;
729
730 coreid_mask |= VIDC_CORE_ID_2;
731 }
732
733 ret = poweroff_coreid(core, coreid_mask);
734 if (ret)
735 return ret;
736
737 done:
738 inst->clk_data.core_id = VIDC_CORE_ID_DEFAULT;
739 inst->core_acquired = false;
740 return 0;
741 }
742
coreid_power_v4(struct venus_inst * inst,int on)743 static int coreid_power_v4(struct venus_inst *inst, int on)
744 {
745 struct venus_core *core = inst->core;
746 int ret;
747
748 if (legacy_binding)
749 return 0;
750
751 if (on == POWER_ON) {
752 ret = decide_core(inst);
753 if (ret)
754 return ret;
755
756 mutex_lock(&core->lock);
757 ret = acquire_core(inst);
758 mutex_unlock(&core->lock);
759 } else {
760 mutex_lock(&core->lock);
761 ret = release_core(inst);
762 mutex_unlock(&core->lock);
763 }
764
765 return ret;
766 }
767
vdec_get_v4(struct device * dev)768 static int vdec_get_v4(struct device *dev)
769 {
770 struct venus_core *core = dev_get_drvdata(dev);
771
772 if (!legacy_binding)
773 return 0;
774
775 return vcodec_clks_get(core, dev, core->vcodec0_clks,
776 core->res->vcodec0_clks);
777 }
778
vdec_put_v4(struct device * dev)779 static void vdec_put_v4(struct device *dev)
780 {
781 struct venus_core *core = dev_get_drvdata(dev);
782 unsigned int i;
783
784 if (!legacy_binding)
785 return;
786
787 for (i = 0; i < core->res->vcodec_clks_num; i++)
788 core->vcodec0_clks[i] = NULL;
789 }
790
vdec_power_v4(struct device * dev,int on)791 static int vdec_power_v4(struct device *dev, int on)
792 {
793 struct venus_core *core = dev_get_drvdata(dev);
794 int ret;
795
796 if (!legacy_binding)
797 return 0;
798
799 ret = vcodec_control_v4(core, VIDC_CORE_ID_1, true);
800 if (ret)
801 return ret;
802
803 if (on == POWER_ON)
804 ret = vcodec_clks_enable(core, core->vcodec0_clks);
805 else
806 vcodec_clks_disable(core, core->vcodec0_clks);
807
808 vcodec_control_v4(core, VIDC_CORE_ID_1, false);
809
810 return ret;
811 }
812
venc_get_v4(struct device * dev)813 static int venc_get_v4(struct device *dev)
814 {
815 struct venus_core *core = dev_get_drvdata(dev);
816
817 if (!legacy_binding)
818 return 0;
819
820 return vcodec_clks_get(core, dev, core->vcodec1_clks,
821 core->res->vcodec1_clks);
822 }
823
venc_put_v4(struct device * dev)824 static void venc_put_v4(struct device *dev)
825 {
826 struct venus_core *core = dev_get_drvdata(dev);
827 unsigned int i;
828
829 if (!legacy_binding)
830 return;
831
832 for (i = 0; i < core->res->vcodec_clks_num; i++)
833 core->vcodec1_clks[i] = NULL;
834 }
835
venc_power_v4(struct device * dev,int on)836 static int venc_power_v4(struct device *dev, int on)
837 {
838 struct venus_core *core = dev_get_drvdata(dev);
839 int ret;
840
841 if (!legacy_binding)
842 return 0;
843
844 ret = vcodec_control_v4(core, VIDC_CORE_ID_2, true);
845 if (ret)
846 return ret;
847
848 if (on == POWER_ON)
849 ret = vcodec_clks_enable(core, core->vcodec1_clks);
850 else
851 vcodec_clks_disable(core, core->vcodec1_clks);
852
853 vcodec_control_v4(core, VIDC_CORE_ID_2, false);
854
855 return ret;
856 }
857
vcodec_domains_get(struct venus_core * core)858 static int vcodec_domains_get(struct venus_core *core)
859 {
860 int ret;
861 struct device **opp_virt_dev;
862 struct device *dev = core->dev;
863 const struct venus_resources *res = core->res;
864 struct device *pd;
865 unsigned int i;
866
867 if (!res->vcodec_pmdomains_num)
868 goto skip_pmdomains;
869
870 for (i = 0; i < res->vcodec_pmdomains_num; i++) {
871 pd = dev_pm_domain_attach_by_name(dev,
872 res->vcodec_pmdomains[i]);
873 if (IS_ERR(pd))
874 return PTR_ERR(pd);
875 core->pmdomains[i] = pd;
876 }
877
878 skip_pmdomains:
879 if (!core->has_opp_table)
880 return 0;
881
882 /* Attach the power domain for setting performance state */
883 ret = devm_pm_opp_attach_genpd(dev, res->opp_pmdomain, &opp_virt_dev);
884 if (ret)
885 goto opp_attach_err;
886
887 core->opp_pmdomain = *opp_virt_dev;
888 core->opp_dl_venus = device_link_add(dev, core->opp_pmdomain,
889 DL_FLAG_RPM_ACTIVE |
890 DL_FLAG_PM_RUNTIME |
891 DL_FLAG_STATELESS);
892 if (!core->opp_dl_venus) {
893 ret = -ENODEV;
894 goto opp_attach_err;
895 }
896
897 return 0;
898
899 opp_attach_err:
900 for (i = 0; i < res->vcodec_pmdomains_num; i++) {
901 if (IS_ERR_OR_NULL(core->pmdomains[i]))
902 continue;
903 dev_pm_domain_detach(core->pmdomains[i], true);
904 }
905
906 return ret;
907 }
908
vcodec_domains_put(struct venus_core * core)909 static void vcodec_domains_put(struct venus_core *core)
910 {
911 const struct venus_resources *res = core->res;
912 unsigned int i;
913
914 if (!res->vcodec_pmdomains_num)
915 goto skip_pmdomains;
916
917 for (i = 0; i < res->vcodec_pmdomains_num; i++) {
918 if (IS_ERR_OR_NULL(core->pmdomains[i]))
919 continue;
920 dev_pm_domain_detach(core->pmdomains[i], true);
921 }
922
923 skip_pmdomains:
924 if (!core->has_opp_table)
925 return;
926
927 if (core->opp_dl_venus)
928 device_link_del(core->opp_dl_venus);
929 }
930
core_resets_reset(struct venus_core * core)931 static int core_resets_reset(struct venus_core *core)
932 {
933 const struct venus_resources *res = core->res;
934 unsigned int i;
935 int ret;
936
937 if (!res->resets_num)
938 return 0;
939
940 for (i = 0; i < res->resets_num; i++) {
941 ret = reset_control_assert(core->resets[i]);
942 if (ret)
943 goto err;
944
945 usleep_range(150, 250);
946 ret = reset_control_deassert(core->resets[i]);
947 if (ret)
948 goto err;
949 }
950
951 err:
952 return ret;
953 }
954
core_resets_get(struct venus_core * core)955 static int core_resets_get(struct venus_core *core)
956 {
957 struct device *dev = core->dev;
958 const struct venus_resources *res = core->res;
959 unsigned int i;
960 int ret;
961
962 if (!res->resets_num)
963 return 0;
964
965 for (i = 0; i < res->resets_num; i++) {
966 core->resets[i] =
967 devm_reset_control_get_exclusive(dev, res->resets[i]);
968 if (IS_ERR(core->resets[i])) {
969 ret = PTR_ERR(core->resets[i]);
970 return ret;
971 }
972 }
973
974 return 0;
975 }
976
core_get_v4(struct venus_core * core)977 static int core_get_v4(struct venus_core *core)
978 {
979 struct device *dev = core->dev;
980 const struct venus_resources *res = core->res;
981 int ret;
982
983 ret = core_clks_get(core);
984 if (ret)
985 return ret;
986
987 if (!res->vcodec_pmdomains_num)
988 legacy_binding = true;
989
990 dev_info(dev, "%s legacy binding\n", legacy_binding ? "" : "non");
991
992 ret = vcodec_clks_get(core, dev, core->vcodec0_clks, res->vcodec0_clks);
993 if (ret)
994 return ret;
995
996 ret = vcodec_clks_get(core, dev, core->vcodec1_clks, res->vcodec1_clks);
997 if (ret)
998 return ret;
999
1000 ret = core_resets_get(core);
1001 if (ret)
1002 return ret;
1003
1004 if (legacy_binding)
1005 return 0;
1006
1007 ret = devm_pm_opp_set_clkname(dev, "core");
1008 if (ret)
1009 return ret;
1010
1011 if (core->res->opp_pmdomain) {
1012 ret = devm_pm_opp_of_add_table(dev);
1013 if (!ret) {
1014 core->has_opp_table = true;
1015 } else if (ret != -ENODEV) {
1016 dev_err(dev, "invalid OPP table in device tree\n");
1017 return ret;
1018 }
1019 }
1020
1021 ret = vcodec_domains_get(core);
1022 if (ret)
1023 return ret;
1024
1025 return 0;
1026 }
1027
core_put_v4(struct venus_core * core)1028 static void core_put_v4(struct venus_core *core)
1029 {
1030 if (legacy_binding)
1031 return;
1032
1033 vcodec_domains_put(core);
1034 }
1035
core_power_v4(struct venus_core * core,int on)1036 static int core_power_v4(struct venus_core *core, int on)
1037 {
1038 struct device *dev = core->dev;
1039 struct device *pmctrl = core->pmdomains[0];
1040 int ret = 0;
1041
1042 if (on == POWER_ON) {
1043 if (pmctrl) {
1044 ret = pm_runtime_resume_and_get(pmctrl);
1045 if (ret < 0) {
1046 return ret;
1047 }
1048 }
1049
1050 ret = core_resets_reset(core);
1051 if (ret) {
1052 if (pmctrl)
1053 pm_runtime_put_sync(pmctrl);
1054 return ret;
1055 }
1056
1057 ret = core_clks_enable(core);
1058 if (ret < 0 && pmctrl)
1059 pm_runtime_put_sync(pmctrl);
1060 } else {
1061 /* Drop the performance state vote */
1062 if (core->opp_pmdomain)
1063 dev_pm_opp_set_rate(dev, 0);
1064
1065 core_clks_disable(core);
1066
1067 ret = core_resets_reset(core);
1068
1069 if (pmctrl)
1070 pm_runtime_put_sync(pmctrl);
1071 }
1072
1073 return ret;
1074 }
1075
calculate_inst_freq(struct venus_inst * inst,unsigned long filled_len)1076 static unsigned long calculate_inst_freq(struct venus_inst *inst,
1077 unsigned long filled_len)
1078 {
1079 unsigned long vpp_freq_per_mb = 0, vpp_freq = 0, vsp_freq = 0;
1080 u32 fps = (u32)inst->fps;
1081 u32 mbs_per_sec;
1082
1083 mbs_per_sec = load_per_instance(inst);
1084
1085 if (inst->state != INST_START)
1086 return 0;
1087
1088 if (inst->session_type == VIDC_SESSION_TYPE_ENC)
1089 vpp_freq_per_mb = inst->flags & VENUS_LOW_POWER ?
1090 inst->clk_data.low_power_freq :
1091 inst->clk_data.vpp_freq;
1092
1093 vpp_freq = mbs_per_sec * vpp_freq_per_mb;
1094 /* 21 / 20 is overhead factor */
1095 vpp_freq += vpp_freq / 20;
1096 vsp_freq = mbs_per_sec * inst->clk_data.vsp_freq;
1097
1098 /* 10 / 7 is overhead factor */
1099 if (inst->session_type == VIDC_SESSION_TYPE_ENC)
1100 vsp_freq += (inst->controls.enc.bitrate * 10) / 7;
1101 else
1102 vsp_freq += ((fps * filled_len * 8) * 10) / 7;
1103
1104 return max(vpp_freq, vsp_freq);
1105 }
1106
load_scale_v4(struct venus_inst * inst)1107 static int load_scale_v4(struct venus_inst *inst)
1108 {
1109 struct venus_core *core = inst->core;
1110 const struct freq_tbl *table = core->res->freq_tbl;
1111 unsigned int num_rows = core->res->freq_tbl_size;
1112 struct device *dev = core->dev;
1113 unsigned long freq = 0, freq_core1 = 0, freq_core2 = 0;
1114 unsigned long filled_len = 0;
1115 int i, ret;
1116
1117 for (i = 0; i < inst->num_input_bufs; i++)
1118 filled_len = max(filled_len, inst->payloads[i]);
1119
1120 if (inst->session_type == VIDC_SESSION_TYPE_DEC && !filled_len)
1121 return 0;
1122
1123 freq = calculate_inst_freq(inst, filled_len);
1124 inst->clk_data.freq = freq;
1125
1126 mutex_lock(&core->lock);
1127 list_for_each_entry(inst, &core->instances, list) {
1128 if (inst->clk_data.core_id == VIDC_CORE_ID_1) {
1129 freq_core1 += inst->clk_data.freq;
1130 } else if (inst->clk_data.core_id == VIDC_CORE_ID_2) {
1131 freq_core2 += inst->clk_data.freq;
1132 } else if (inst->clk_data.core_id == VIDC_CORE_ID_3) {
1133 freq_core1 += inst->clk_data.freq;
1134 freq_core2 += inst->clk_data.freq;
1135 }
1136 }
1137 mutex_unlock(&core->lock);
1138
1139 freq = max(freq_core1, freq_core2);
1140
1141 if (freq > table[0].freq) {
1142 freq = table[0].freq;
1143 dev_warn(dev, "HW is overloaded, needed: %lu max: %lu\n",
1144 freq, table[0].freq);
1145 goto set_freq;
1146 }
1147
1148 for (i = num_rows - 1 ; i >= 0; i--) {
1149 if (freq <= table[i].freq) {
1150 freq = table[i].freq;
1151 break;
1152 }
1153 }
1154
1155 set_freq:
1156
1157 ret = core_clks_set_rate(core, freq);
1158 if (ret) {
1159 dev_err(dev, "failed to set clock rate %lu (%d)\n",
1160 freq, ret);
1161 return ret;
1162 }
1163
1164 ret = load_scale_bw(core);
1165 if (ret) {
1166 dev_err(dev, "failed to set bandwidth (%d)\n",
1167 ret);
1168 return ret;
1169 }
1170
1171 return 0;
1172 }
1173
1174 static const struct venus_pm_ops pm_ops_v4 = {
1175 .core_get = core_get_v4,
1176 .core_put = core_put_v4,
1177 .core_power = core_power_v4,
1178 .vdec_get = vdec_get_v4,
1179 .vdec_put = vdec_put_v4,
1180 .vdec_power = vdec_power_v4,
1181 .venc_get = venc_get_v4,
1182 .venc_put = venc_put_v4,
1183 .venc_power = venc_power_v4,
1184 .coreid_power = coreid_power_v4,
1185 .load_scale = load_scale_v4,
1186 };
1187
venus_pm_get(enum hfi_version version)1188 const struct venus_pm_ops *venus_pm_get(enum hfi_version version)
1189 {
1190 switch (version) {
1191 case HFI_VERSION_1XX:
1192 default:
1193 return &pm_ops_v1;
1194 case HFI_VERSION_3XX:
1195 return &pm_ops_v3;
1196 case HFI_VERSION_4XX:
1197 case HFI_VERSION_6XX:
1198 return &pm_ops_v4;
1199 }
1200
1201 return NULL;
1202 }
1203
