1 /*
2 * Copyright (c) 2018-2021 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #define DT_DRV_COMPAT intel_hpet
8 #include <zephyr/device.h>
9 #include <zephyr/drivers/timer/system_timer.h>
10 #include <zephyr/sys_clock.h>
11 #include <zephyr/spinlock.h>
12 #include <zephyr/irq.h>
13 #include <zephyr/linker/sections.h>
14
15 #include <zephyr/dt-bindings/interrupt-controller/intel-ioapic.h>
16
17 #include <soc.h>
18
19 /**
20 * @file
21 * @brief HPET (High Precision Event Timers) driver
22 *
23 * HPET hardware contains a number of timers which can be used by
24 * the operating system, where the number of timers is implementation
25 * specific. The timers are implemented as a single up-counter with
26 * a set of comparators where the counter increases monotonically.
27 * Each timer has a match register and a comparator, and can generate
28 * an interrupt when the value in the match register equals the value of
29 * the free running counter. Some of these timers can be enabled to
30 * generate periodic interrupt.
31 *
32 * The HPET registers are usually mapped to memory space on x86
33 * hardware. If this is not the case, custom register access functions
34 * can be used by defining macro HPET_USE_CUSTOM_REG_ACCESS_FUNCS in
35 * soc.h, and implementing necessary initialization and access
36 * functions as described below.
37 *
38 * HPET_COUNTER_CLK_PERIOD can be overridden in soc.h if
39 * COUNTER_CLK_PERIOD is not in femtoseconds (1e-15 sec).
40 */
41
42 /* General Configuration register */
43 #define GCONF_ENABLE BIT(0)
44 #define GCONF_LR BIT(1) /* legacy interrupt routing, */
45 /* disables PIT */
46
47 /* General Interrupt Status register */
48 #define TIMER0_INT_STS BIT(0)
49
50 /* Timer Configuration and Capabilities register */
51 #define TIMER_CONF_INT_LEVEL BIT(1)
52 #define TIMER_CONF_INT_ENABLE BIT(2)
53 #define TIMER_CONF_PERIODIC BIT(3)
54 #define TIMER_CONF_VAL_SET BIT(6)
55 #define TIMER_CONF_MODE32 BIT(8)
56 #define TIMER_CONF_FSB_EN BIT(14) /* FSB interrupt delivery */
57 /* enable */
58
59 DEVICE_MMIO_TOPLEVEL_STATIC(hpet_regs, DT_DRV_INST(0));
60
61 #define HPET_REG_ADDR(off) \
62 ((mm_reg_t)(DEVICE_MMIO_TOPLEVEL_GET(hpet_regs) + (off)))
63
64 /* High dword of General Capabilities and ID register */
65 #define CLK_PERIOD_REG HPET_REG_ADDR(0x04)
66
67 /* General Configuration register */
68 #define GCONF_REG HPET_REG_ADDR(0x10)
69
70 /* General Interrupt Status register */
71 #define INTR_STATUS_REG HPET_REG_ADDR(0x20)
72
73 /* Main Counter Register */
74 #define MAIN_COUNTER_LOW_REG HPET_REG_ADDR(0xf0)
75 #define MAIN_COUNTER_HIGH_REG HPET_REG_ADDR(0xf4)
76
77 /* Timer 0 Configuration and Capabilities register */
78 #define TIMER0_CONF_REG HPET_REG_ADDR(0x100)
79
80 /* Timer 0 Comparator Register */
81 #define TIMER0_COMPARATOR_LOW_REG HPET_REG_ADDR(0x108)
82 #define TIMER0_COMPARATOR_HIGH_REG HPET_REG_ADDR(0x10c)
83
84 #if defined(CONFIG_TEST)
85 const int32_t z_sys_timer_irq_for_test = DT_IRQN(DT_INST(0, intel_hpet));
86 #endif
87
88 /**
89 * @brief Return the value of the main counter.
90 *
91 * @return Value of Main Counter
92 */
hpet_counter_get(void)93 static inline uint64_t hpet_counter_get(void)
94 {
95 #ifdef CONFIG_64BIT
96 uint64_t val = sys_read64(MAIN_COUNTER_LOW_REG);
97
98 return val;
99 #else
100 uint32_t high;
101 uint32_t low;
102
103 do {
104 high = sys_read32(MAIN_COUNTER_HIGH_REG);
105 low = sys_read32(MAIN_COUNTER_LOW_REG);
106 } while (high != sys_read32(MAIN_COUNTER_HIGH_REG));
107
108 return ((uint64_t)high << 32) | low;
109 #endif
110 }
111
112 /**
113 * @brief Get COUNTER_CLK_PERIOD
114 *
115 * Read and return the COUNTER_CLK_PERIOD, which is the high
116 * 32-bit of the General Capabilities and ID Register. This can
117 * be used to calculate the frequency of the main counter.
118 *
119 * Usually the period is in femtoseconds. If this is not
120 * the case, define HPET_COUNTER_CLK_PERIOD in soc.h so
121 * it can be used to calculate frequency.
122 *
123 * @return COUNTER_CLK_PERIOD
124 */
hpet_counter_clk_period_get(void)125 static inline uint32_t hpet_counter_clk_period_get(void)
126 {
127 return sys_read32(CLK_PERIOD_REG);
128 }
129
130 /**
131 * @brief Return the value of the General Configuration Register
132 *
133 * @return Value of the General Configuration Register
134 */
hpet_gconf_get(void)135 static inline uint32_t hpet_gconf_get(void)
136 {
137 return sys_read32(GCONF_REG);
138 }
139
140 /**
141 * @brief Write to General Configuration Register
142 *
143 * @param val Value to be written to the register
144 */
hpet_gconf_set(uint32_t val)145 static inline void hpet_gconf_set(uint32_t val)
146 {
147 sys_write32(val, GCONF_REG);
148 }
149
150 /**
151 * @brief Return the value of the Timer Configuration Register
152 *
153 * This reads and returns the value of the Timer Configuration
154 * Register of Timer #0.
155 *
156 * @return Value of the Timer Configuration Register
157 */
hpet_timer_conf_get(void)158 static inline uint32_t hpet_timer_conf_get(void)
159 {
160 return sys_read32(TIMER0_CONF_REG);
161 }
162
163 /**
164 * @brief Write to the Timer Configuration Register
165 *
166 * This writes the specified value to the Timer Configuration
167 * Register of Timer #0.
168 *
169 * @param val Value to be written to the register
170 */
hpet_timer_conf_set(uint32_t val)171 static inline void hpet_timer_conf_set(uint32_t val)
172 {
173 sys_write32(val, TIMER0_CONF_REG);
174 }
175
176 /*
177 * The following register access functions should work on generic x86
178 * hardware. If the targeted SoC requires special handling of HPET
179 * registers, these functions will need to be implemented in the SoC
180 * layer by first defining the macro HPET_USE_CUSTOM_REG_ACCESS_FUNCS
181 * in soc.h to signal such intent.
182 *
183 * This is a list of functions which must be implemented in the SoC
184 * layer:
185 * void hpet_timer_comparator_set(uint32_t val)
186 */
187 #ifndef HPET_USE_CUSTOM_REG_ACCESS_FUNCS
188
189 /**
190 * @brief Write to the Timer Comparator Value Register
191 *
192 * This writes the specified value to the Timer Comparator
193 * Value Register of Timer #0.
194 *
195 * @param val Value to be written to the register
196 */
hpet_timer_comparator_set(uint64_t val)197 static inline void hpet_timer_comparator_set(uint64_t val)
198 {
199 #if CONFIG_X86_64
200 sys_write64(val, TIMER0_COMPARATOR_LOW_REG);
201 #else
202 sys_write32((uint32_t)val, TIMER0_COMPARATOR_LOW_REG);
203 sys_write32((uint32_t)(val >> 32), TIMER0_COMPARATOR_HIGH_REG);
204 #endif
205 }
206 #endif /* HPET_USE_CUSTOM_REG_ACCESS_FUNCS */
207
208 #ifndef HPET_COUNTER_CLK_PERIOD
209 /* COUNTER_CLK_PERIOD (CLK_PERIOD_REG) is in femtoseconds (1e-15 sec) */
210 #define HPET_COUNTER_CLK_PERIOD (1000000000000000ULL)
211 #endif
212
213 /*
214 * HPET_INT_LEVEL_TRIGGER is used to set HPET interrupt as level trigger
215 * for ARM CPU with NVIC like EHL PSE, whose DTS interrupt setting
216 * has no "sense" cell.
217 */
218 #if (DT_INST_IRQ_HAS_CELL(0, sense))
219 #ifdef HPET_INT_LEVEL_TRIGGER
220 __WARN("HPET_INT_LEVEL_TRIGGER has no effect, DTS setting is used instead")
221 #undef HPET_INT_LEVEL_TRIGGER
222 #endif
223 #if ((DT_INST_IRQ(0, sense) & IRQ_TYPE_LEVEL) == IRQ_TYPE_LEVEL)
224 #define HPET_INT_LEVEL_TRIGGER
225 #endif
226 #endif /* (DT_INST_IRQ_HAS_CELL(0, sense)) */
227
228 static __pinned_bss struct k_spinlock lock;
229 static __pinned_bss uint64_t last_count;
230 static __pinned_bss uint64_t last_tick;
231 static __pinned_bss uint32_t last_elapsed;
232
233 #ifdef CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME
234 static __pinned_bss unsigned int cyc_per_tick;
235 #else
236 #define cyc_per_tick \
237 (CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC / CONFIG_SYS_CLOCK_TICKS_PER_SEC)
238 #endif /* CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME */
239
240 #define HPET_MAX_TICKS ((int32_t)0x7fffffff)
241
242 #ifdef HPET_INT_LEVEL_TRIGGER
243 /**
244 * @brief Write to General Interrupt Status Register
245 *
246 * This is used to acknowledge and clear interrupt bits.
247 *
248 * @param val Value to be written to the register
249 */
hpet_int_sts_set(uint32_t val)250 static inline void hpet_int_sts_set(uint32_t val)
251 {
252 sys_write32(val, INTR_STATUS_REG);
253 }
254 #endif
255
256 /* ensure the comparator is always set ahead of the current counter value */
hpet_timer_comparator_set_safe(uint64_t next)257 static inline void hpet_timer_comparator_set_safe(uint64_t next)
258 {
259 hpet_timer_comparator_set(next);
260
261 uint64_t now = hpet_counter_get();
262
263 if (unlikely((int64_t)(next - now) <= 0)) {
264 uint32_t bump = 1;
265
266 do {
267 next = now + bump;
268 bump *= 2;
269 hpet_timer_comparator_set(next);
270 now = hpet_counter_get();
271 } while ((int64_t)(next - now) <= 0);
272 }
273 }
274
275 __isr
hpet_isr(const void * arg)276 static void hpet_isr(const void *arg)
277 {
278 ARG_UNUSED(arg);
279
280 k_spinlock_key_t key = k_spin_lock(&lock);
281
282 uint64_t now = hpet_counter_get();
283
284 #ifdef HPET_INT_LEVEL_TRIGGER
285 /*
286 * Clear interrupt only if level trigger is selected.
287 * When edge trigger is selected, spec says only 0 can
288 * be written.
289 */
290 hpet_int_sts_set(TIMER0_INT_STS);
291 #endif
292
293 if (IS_ENABLED(CONFIG_SMP) &&
294 IS_ENABLED(CONFIG_QEMU_TARGET)) {
295 /* Qemu in SMP mode has observed the clock going
296 * "backwards" relative to interrupts already received
297 * on the other CPU, despite the HPET being
298 * theoretically a global device.
299 */
300 int64_t diff = (int64_t)(now - last_count);
301
302 if (last_count && diff < 0) {
303 now = last_count;
304 }
305 }
306 uint32_t dticks = (uint32_t)((now - last_count) / cyc_per_tick);
307
308 last_count += (uint64_t)dticks * cyc_per_tick;
309 last_tick += dticks;
310 last_elapsed = 0;
311
312 if (!IS_ENABLED(CONFIG_TICKLESS_KERNEL)) {
313 uint64_t next = last_count + cyc_per_tick;
314
315 hpet_timer_comparator_set_safe(next);
316 }
317
318 k_spin_unlock(&lock, key);
319 sys_clock_announce(dticks);
320 }
321
322 __pinned_func
config_timer0(unsigned int irq)323 static void config_timer0(unsigned int irq)
324 {
325 uint32_t val = hpet_timer_conf_get();
326
327 /* 5-bit IRQ field starting at bit 9 */
328 val = (val & ~(0x1f << 9)) | ((irq & 0x1f) << 9);
329
330 #ifdef HPET_INT_LEVEL_TRIGGER
331 /* Set level trigger if selected */
332 val |= TIMER_CONF_INT_LEVEL;
333 #endif
334
335 val &= ~((uint32_t)(TIMER_CONF_MODE32 | TIMER_CONF_PERIODIC |
336 TIMER_CONF_FSB_EN));
337 val |= TIMER_CONF_INT_ENABLE;
338
339 hpet_timer_conf_set(val);
340 }
341
342 __boot_func
smp_timer_init(void)343 void smp_timer_init(void)
344 {
345 /* Noop, the HPET is a single system-wide device and it's
346 * configured to deliver interrupts to every CPU, so there's
347 * nothing to do at initialization on auxiliary CPUs.
348 */
349 }
350
351 __pinned_func
sys_clock_set_timeout(int32_t ticks,bool idle)352 void sys_clock_set_timeout(int32_t ticks, bool idle)
353 {
354 ARG_UNUSED(idle);
355
356 #if defined(CONFIG_TICKLESS_KERNEL)
357 uint32_t reg;
358
359 if (ticks == K_TICKS_FOREVER && idle) {
360 reg = hpet_gconf_get();
361 reg &= ~GCONF_ENABLE;
362 hpet_gconf_set(reg);
363 return;
364 }
365
366 ticks = ticks == K_TICKS_FOREVER ? HPET_MAX_TICKS : ticks;
367 ticks = CLAMP(ticks, 0, HPET_MAX_TICKS/2);
368
369 k_spinlock_key_t key = k_spin_lock(&lock);
370 uint64_t cyc = (last_tick + last_elapsed + ticks) * cyc_per_tick;
371
372 hpet_timer_comparator_set_safe(cyc);
373 k_spin_unlock(&lock, key);
374 #endif
375 }
376
377 __pinned_func
sys_clock_elapsed(void)378 uint32_t sys_clock_elapsed(void)
379 {
380 if (!IS_ENABLED(CONFIG_TICKLESS_KERNEL)) {
381 return 0;
382 }
383
384 k_spinlock_key_t key = k_spin_lock(&lock);
385 uint64_t now = hpet_counter_get();
386 uint32_t ret = (uint32_t)((now - last_count) / cyc_per_tick);
387
388 last_elapsed = ret;
389 k_spin_unlock(&lock, key);
390 return ret;
391 }
392
393 __pinned_func
sys_clock_cycle_get_32(void)394 uint32_t sys_clock_cycle_get_32(void)
395 {
396 return (uint32_t)hpet_counter_get();
397 }
398
399 __pinned_func
sys_clock_cycle_get_64(void)400 uint64_t sys_clock_cycle_get_64(void)
401 {
402 return hpet_counter_get();
403 }
404
405 __pinned_func
sys_clock_idle_exit(void)406 void sys_clock_idle_exit(void)
407 {
408 uint32_t reg;
409
410 reg = hpet_gconf_get();
411 reg |= GCONF_ENABLE;
412 hpet_gconf_set(reg);
413 }
414
415 __boot_func
sys_clock_driver_init(void)416 static int sys_clock_driver_init(void)
417 {
418 extern int z_clock_hw_cycles_per_sec;
419 uint32_t hz, reg;
420
421 ARG_UNUSED(hz);
422 ARG_UNUSED(z_clock_hw_cycles_per_sec);
423
424 DEVICE_MMIO_TOPLEVEL_MAP(hpet_regs, K_MEM_CACHE_NONE);
425
426 #if DT_INST_IRQ_HAS_CELL(0, sense)
427 IRQ_CONNECT(DT_INST_IRQN(0),
428 DT_INST_IRQ(0, priority),
429 hpet_isr, 0, DT_INST_IRQ(0, sense));
430 #else
431 IRQ_CONNECT(DT_INST_IRQN(0),
432 DT_INST_IRQ(0, priority),
433 hpet_isr, 0, 0);
434 #endif
435 config_timer0(DT_INST_IRQN(0));
436 irq_enable(DT_INST_IRQN(0));
437
438 #ifdef CONFIG_TIMER_READS_ITS_FREQUENCY_AT_RUNTIME
439 hz = (uint32_t)(HPET_COUNTER_CLK_PERIOD / hpet_counter_clk_period_get());
440 z_clock_hw_cycles_per_sec = hz;
441 cyc_per_tick = hz / CONFIG_SYS_CLOCK_TICKS_PER_SEC;
442 #endif
443
444 reg = hpet_gconf_get();
445 reg |= GCONF_ENABLE;
446
447 #if (DT_INST_PROP(0, no_legacy_irq) == 0)
448 /* Note: we set the legacy routing bit, because otherwise
449 * nothing in Zephyr disables the PIT which then fires
450 * interrupts into the same IRQ. But that means we're then
451 * forced to use IRQ2 contra the way the kconfig IRQ selection
452 * is supposed to work. Should fix this.
453 */
454 reg |= GCONF_LR;
455 #endif
456
457 hpet_gconf_set(reg);
458
459 last_tick = hpet_counter_get() / cyc_per_tick;
460 last_count = last_tick * cyc_per_tick;
461 hpet_timer_comparator_set_safe(last_count + cyc_per_tick);
462
463 return 0;
464 }
465
466 SYS_INIT(sys_clock_driver_init, PRE_KERNEL_2,
467 CONFIG_SYSTEM_CLOCK_INIT_PRIORITY);
468
