RTEMS
kern_tc.c
1 /*-
2  * ----------------------------------------------------------------------------
3  * "THE BEER-WARE LICENSE" (Revision 42):
4  * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you
5  * can do whatever you want with this stuff. If we meet some day, and you think
6  * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
7  * ----------------------------------------------------------------------------
8  *
9  * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation
10  * All rights reserved.
11  *
12  * Portions of this software were developed by Julien Ridoux at the University
13  * of Melbourne under sponsorship from the FreeBSD Foundation.
14  *
15  * Portions of this software were developed by Konstantin Belousov
16  * under sponsorship from the FreeBSD Foundation.
17  */
18 
19 #ifdef __rtems__
20 #include <sys/lock.h>
21 #define _KERNEL
22 #define binuptime(_bt) _Timecounter_Binuptime(_bt)
23 #define nanouptime(_tsp) _Timecounter_Nanouptime(_tsp)
24 #define microuptime(_tvp) _Timecounter_Microuptime(_tvp)
25 #define bintime(_bt) _Timecounter_Bintime(_bt)
26 #define nanotime(_tsp) _Timecounter_Nanotime(_tsp)
27 #define microtime(_tvp) _Timecounter_Microtime(_tvp)
28 #define getbinuptime(_bt) _Timecounter_Getbinuptime(_bt)
29 #define getnanouptime(_tsp) _Timecounter_Getnanouptime(_tsp)
30 #define getmicrouptime(_tvp) _Timecounter_Getmicrouptime(_tvp)
31 #define getbintime(_bt) _Timecounter_Getbintime(_bt)
32 #define getnanotime(_tsp) _Timecounter_Getnanotime(_tsp)
33 #define getmicrotime(_tvp) _Timecounter_Getmicrotime(_tvp)
34 #define getboottime(_tvp) _Timecounter_Getboottime(_tvp)
35 #define getboottimebin(_bt) _Timecounter_Getboottimebin(_bt)
36 #define tc_init _Timecounter_Install
37 #define timecounter _Timecounter
38 #define time_second _Timecounter_Time_second
39 #define time_uptime _Timecounter_Time_uptime
41 #include <rtems/score/atomic.h>
42 #include <rtems/score/smp.h>
43 #include <rtems/score/todimpl.h>
45 #endif /* __rtems__ */
46 #include <sys/cdefs.h>
47 __FBSDID("$FreeBSD: head/sys/kern/kern_tc.c 324528 2017-10-11 11:03:11Z kib $");
48 
49 #include "opt_compat.h"
50 #include "opt_ntp.h"
51 #include "opt_ffclock.h"
52 
53 #include <sys/param.h>
54 #ifndef __rtems__
55 #include <sys/kernel.h>
56 #include <sys/limits.h>
57 #include <sys/lock.h>
58 #include <sys/mutex.h>
59 #include <sys/proc.h>
60 #include <sys/sbuf.h>
61 #include <sys/sleepqueue.h>
62 #include <sys/sysctl.h>
63 #include <sys/syslog.h>
64 #include <sys/systm.h>
65 #endif /* __rtems__ */
66 #include <sys/timeffc.h>
67 #include <sys/timepps.h>
68 #include <sys/timetc.h>
69 #include <sys/timex.h>
70 #ifndef __rtems__
71 #include <sys/vdso.h>
72 #endif /* __rtems__ */
73 #ifdef __rtems__
74 #include <limits.h>
75 #include <string.h>
76 #include <rtems.h>
77 ISR_LOCK_DEFINE(, _Timecounter_Lock, "Timecounter")
78 #define _Timecounter_Release(lock_context) \
79  _ISR_lock_Release_and_ISR_enable(&_Timecounter_Lock, lock_context)
80 #define hz rtems_clock_get_ticks_per_second()
81 #define printf(...)
82 #define bcopy(x, y, z) memcpy(y, x, z);
83 #define log(...)
84 static inline int
85 builtin_fls(int x)
86 {
87  return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
88 }
89 #define fls(x) builtin_fls(x)
90 /* FIXME: https://devel.rtems.org/ticket/2348 */
91 #define ntp_update_second(a, b) do { (void) a; (void) b; } while (0)
92 
93 static inline void
94 atomic_thread_fence_acq(void)
95 {
96 
97  _Atomic_Fence(ATOMIC_ORDER_ACQUIRE);
98 }
99 
100 static inline void
101 atomic_thread_fence_rel(void)
102 {
103 
104  _Atomic_Fence(ATOMIC_ORDER_RELEASE);
105 }
106 
107 static inline u_int
108 atomic_load_acq_int(Atomic_Uint *i)
109 {
110 
111  return (_Atomic_Load_uint(i, ATOMIC_ORDER_ACQUIRE));
112 }
113 
114 static inline void
115 atomic_store_rel_int(Atomic_Uint *i, u_int val)
116 {
117 
118  _Atomic_Store_uint(i, val, ATOMIC_ORDER_RELEASE);
119 }
120 #endif /* __rtems__ */
121 
122 /*
123  * A large step happens on boot. This constant detects such steps.
124  * It is relatively small so that ntp_update_second gets called enough
125  * in the typical 'missed a couple of seconds' case, but doesn't loop
126  * forever when the time step is large.
127  */
128 #define LARGE_STEP 200
129 
130 /*
131  * Implement a dummy timecounter which we can use until we get a real one
132  * in the air. This allows the console and other early stuff to use
133  * time services.
134  */
135 
136 static uint32_t
137 dummy_get_timecount(struct timecounter *tc)
138 {
139 #ifndef __rtems__
140  static uint32_t now;
141 
142  return (++now);
143 #else /* __rtems__ */
144  return 0;
145 #endif /* __rtems__ */
146 }
147 
148 static struct timecounter dummy_timecounter = {
149 #ifndef __rtems__
150  dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000
151 #else /* __rtems__ */
152  dummy_get_timecount, ~(uint32_t)0, 1000000, "dummy", -1000000
153 #endif /* __rtems__ */
154 };
155 
156 struct timehands {
157  /* These fields must be initialized by the driver. */
158  struct timecounter *th_counter;
159  int64_t th_adjustment;
160  uint64_t th_scale;
161  uint32_t th_offset_count;
162  struct bintime th_offset;
163  struct bintime th_bintime;
164  struct timeval th_microtime;
165  struct timespec th_nanotime;
166  struct bintime th_boottime;
167  /* Fields not to be copied in tc_windup start with th_generation. */
168 #ifndef __rtems__
169  u_int th_generation;
170 #else /* __rtems__ */
171  Atomic_Uint th_generation;
172 #endif /* __rtems__ */
173  struct timehands *th_next;
174 };
175 
176 #if defined(RTEMS_SMP)
177 static struct timehands th0;
178 static struct timehands th1 = {
179  .th_next = &th0
180 };
181 #endif
182 static struct timehands th0 = {
183  .th_counter = &dummy_timecounter,
184  .th_scale = (uint64_t)-1 / 1000000,
185  .th_offset = { .sec = 1 },
186  .th_generation = 1,
187 #ifdef __rtems__
188  .th_bintime = { .sec = TOD_SECONDS_1970_THROUGH_1988 },
189  .th_microtime = { TOD_SECONDS_1970_THROUGH_1988, 0 },
190  .th_nanotime = { TOD_SECONDS_1970_THROUGH_1988, 0 },
191  .th_boottime = { .sec = TOD_SECONDS_1970_THROUGH_1988 - 1 },
192 #endif /* __rtems__ */
193 #if defined(RTEMS_SMP)
194  .th_next = &th1
195 #else
196  .th_next = &th0
197 #endif
198 };
199 
200 static struct timehands *volatile timehands = &th0;
201 struct timecounter *timecounter = &dummy_timecounter;
202 #ifndef __rtems__
203 static struct timecounter *timecounters = &dummy_timecounter;
204 
205 int tc_min_ticktock_freq = 1;
206 #endif /* __rtems__ */
207 
208 #ifndef __rtems__
209 volatile time_t time_second = 1;
210 volatile time_t time_uptime = 1;
211 #else /* __rtems__ */
212 volatile time_t time_second = TOD_SECONDS_1970_THROUGH_1988;
213 volatile int32_t time_uptime = 1;
214 #endif /* __rtems__ */
215 
216 #ifndef __rtems__
217 static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS);
218 SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD,
219  NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime");
220 
221 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, "");
222 static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
223 
224 static int timestepwarnings;
225 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
226  &timestepwarnings, 0, "Log time steps");
227 
228 struct bintime bt_timethreshold;
229 struct bintime bt_tickthreshold;
230 sbintime_t sbt_timethreshold;
231 sbintime_t sbt_tickthreshold;
232 struct bintime tc_tick_bt;
233 sbintime_t tc_tick_sbt;
234 int tc_precexp;
235 int tc_timepercentage = TC_DEFAULTPERC;
236 static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS);
237 SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation,
238  CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0,
239  sysctl_kern_timecounter_adjprecision, "I",
240  "Allowed time interval deviation in percents");
241 
242 volatile int rtc_generation = 1;
243 
244 static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */
245 #endif /* __rtems__ */
246 
247 static void tc_windup(struct bintime *new_boottimebin);
248 #ifndef __rtems__
249 static void cpu_tick_calibrate(int);
250 #else /* __rtems__ */
251 static void _Timecounter_Windup(struct bintime *new_boottimebin,
252  ISR_lock_Context *lock_context);
253 #endif /* __rtems__ */
254 
255 void dtrace_getnanotime(struct timespec *tsp);
256 
257 #ifndef __rtems__
258 static int
259 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
260 {
261  struct timeval boottime;
262 
263  getboottime(&boottime);
264 
265 #ifndef __mips__
266 #ifdef SCTL_MASK32
267  int tv[2];
268 
269  if (req->flags & SCTL_MASK32) {
270  tv[0] = boottime.tv_sec;
271  tv[1] = boottime.tv_usec;
272  return (SYSCTL_OUT(req, tv, sizeof(tv)));
273  }
274 #endif
275 #endif
276  return (SYSCTL_OUT(req, &boottime, sizeof(boottime)));
277 }
278 
279 static int
280 sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS)
281 {
282  uint32_t ncount;
283  struct timecounter *tc = arg1;
284 
285  ncount = tc->tc_get_timecount(tc);
286  return (sysctl_handle_int(oidp, &ncount, 0, req));
287 }
288 
289 static int
290 sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS)
291 {
292  uint64_t freq;
293  struct timecounter *tc = arg1;
294 
295  freq = tc->tc_frequency;
296  return (sysctl_handle_64(oidp, &freq, 0, req));
297 }
298 #endif /* __rtems__ */
299 
300 /*
301  * Return the difference between the timehands' counter value now and what
302  * was when we copied it to the timehands' offset_count.
303  */
304 static __inline uint32_t
305 tc_delta(struct timehands *th)
306 {
307  struct timecounter *tc;
308 
309  tc = th->th_counter;
310  return ((tc->tc_get_timecount(tc) - th->th_offset_count) &
311  tc->tc_counter_mask);
312 }
313 
314 /*
315  * Functions for reading the time. We have to loop until we are sure that
316  * the timehands that we operated on was not updated under our feet. See
317  * the comment in <sys/time.h> for a description of these 12 functions.
318  */
319 
320 #ifdef FFCLOCK
321 void
322 fbclock_binuptime(struct bintime *bt)
323 {
324  struct timehands *th;
325  unsigned int gen;
326 
327  do {
328  th = timehands;
329  gen = atomic_load_acq_int(&th->th_generation);
330  *bt = th->th_offset;
331  bintime_addx(bt, th->th_scale * tc_delta(th));
332  atomic_thread_fence_acq();
333  } while (gen == 0 || gen != th->th_generation);
334 }
335 
336 void
337 fbclock_nanouptime(struct timespec *tsp)
338 {
339  struct bintime bt;
340 
341  fbclock_binuptime(&bt);
342  bintime2timespec(&bt, tsp);
343 }
344 
345 void
346 fbclock_microuptime(struct timeval *tvp)
347 {
348  struct bintime bt;
349 
350  fbclock_binuptime(&bt);
351  bintime2timeval(&bt, tvp);
352 }
353 
354 void
355 fbclock_bintime(struct bintime *bt)
356 {
357  struct timehands *th;
358  unsigned int gen;
359 
360  do {
361  th = timehands;
362  gen = atomic_load_acq_int(&th->th_generation);
363  *bt = th->th_bintime;
364  bintime_addx(bt, th->th_scale * tc_delta(th));
365  atomic_thread_fence_acq();
366  } while (gen == 0 || gen != th->th_generation);
367 }
368 
369 void
370 fbclock_nanotime(struct timespec *tsp)
371 {
372  struct bintime bt;
373 
374  fbclock_bintime(&bt);
375  bintime2timespec(&bt, tsp);
376 }
377 
378 void
379 fbclock_microtime(struct timeval *tvp)
380 {
381  struct bintime bt;
382 
383  fbclock_bintime(&bt);
384  bintime2timeval(&bt, tvp);
385 }
386 
387 void
388 fbclock_getbinuptime(struct bintime *bt)
389 {
390  struct timehands *th;
391  unsigned int gen;
392 
393  do {
394  th = timehands;
395  gen = atomic_load_acq_int(&th->th_generation);
396  *bt = th->th_offset;
397  atomic_thread_fence_acq();
398  } while (gen == 0 || gen != th->th_generation);
399 }
400 
401 void
402 fbclock_getnanouptime(struct timespec *tsp)
403 {
404  struct timehands *th;
405  unsigned int gen;
406 
407  do {
408  th = timehands;
409  gen = atomic_load_acq_int(&th->th_generation);
410  bintime2timespec(&th->th_offset, tsp);
411  atomic_thread_fence_acq();
412  } while (gen == 0 || gen != th->th_generation);
413 }
414 
415 void
416 fbclock_getmicrouptime(struct timeval *tvp)
417 {
418  struct timehands *th;
419  unsigned int gen;
420 
421  do {
422  th = timehands;
423  gen = atomic_load_acq_int(&th->th_generation);
424  bintime2timeval(&th->th_offset, tvp);
425  atomic_thread_fence_acq();
426  } while (gen == 0 || gen != th->th_generation);
427 }
428 
429 void
430 fbclock_getbintime(struct bintime *bt)
431 {
432  struct timehands *th;
433  unsigned int gen;
434 
435  do {
436  th = timehands;
437  gen = atomic_load_acq_int(&th->th_generation);
438  *bt = th->th_bintime;
439  atomic_thread_fence_acq();
440  } while (gen == 0 || gen != th->th_generation);
441 }
442 
443 void
444 fbclock_getnanotime(struct timespec *tsp)
445 {
446  struct timehands *th;
447  unsigned int gen;
448 
449  do {
450  th = timehands;
451  gen = atomic_load_acq_int(&th->th_generation);
452  *tsp = th->th_nanotime;
453  atomic_thread_fence_acq();
454  } while (gen == 0 || gen != th->th_generation);
455 }
456 
457 void
458 fbclock_getmicrotime(struct timeval *tvp)
459 {
460  struct timehands *th;
461  unsigned int gen;
462 
463  do {
464  th = timehands;
465  gen = atomic_load_acq_int(&th->th_generation);
466  *tvp = th->th_microtime;
467  atomic_thread_fence_acq();
468  } while (gen == 0 || gen != th->th_generation);
469 }
470 #else /* !FFCLOCK */
471 void
472 binuptime(struct bintime *bt)
473 {
474  struct timehands *th;
475  uint32_t gen;
476 
477  do {
478  th = timehands;
479  gen = atomic_load_acq_int(&th->th_generation);
480  *bt = th->th_offset;
481  bintime_addx(bt, th->th_scale * tc_delta(th));
482  atomic_thread_fence_acq();
483  } while (gen == 0 || gen != th->th_generation);
484 }
485 #ifdef __rtems__
486 sbintime_t
488 {
489  struct timehands *th;
490  uint32_t gen;
491  sbintime_t sbt;
492 
493  do {
494  th = timehands;
495  gen = atomic_load_acq_int(&th->th_generation);
496  sbt = bttosbt(th->th_offset);
497  sbt += (th->th_scale * tc_delta(th)) >> 32;
498  atomic_thread_fence_acq();
499  } while (gen == 0 || gen != th->th_generation);
500 
501  return (sbt);
502 }
503 #endif /* __rtems__ */
504 
505 void
506 nanouptime(struct timespec *tsp)
507 {
508  struct bintime bt;
509 
510  binuptime(&bt);
511  bintime2timespec(&bt, tsp);
512 }
513 
514 void
515 microuptime(struct timeval *tvp)
516 {
517  struct bintime bt;
518 
519  binuptime(&bt);
520  bintime2timeval(&bt, tvp);
521 }
522 
523 void
524 bintime(struct bintime *bt)
525 {
526  struct timehands *th;
527  u_int gen;
528 
529  do {
530  th = timehands;
531  gen = atomic_load_acq_int(&th->th_generation);
532  *bt = th->th_bintime;
533  bintime_addx(bt, th->th_scale * tc_delta(th));
534  atomic_thread_fence_acq();
535  } while (gen == 0 || gen != th->th_generation);
536 }
537 
538 void
539 nanotime(struct timespec *tsp)
540 {
541  struct bintime bt;
542 
543  bintime(&bt);
544  bintime2timespec(&bt, tsp);
545 }
546 
547 void
548 microtime(struct timeval *tvp)
549 {
550  struct bintime bt;
551 
552  bintime(&bt);
553  bintime2timeval(&bt, tvp);
554 }
555 
556 void
557 getbinuptime(struct bintime *bt)
558 {
559  struct timehands *th;
560  uint32_t gen;
561 
562  do {
563  th = timehands;
564  gen = atomic_load_acq_int(&th->th_generation);
565  *bt = th->th_offset;
566  atomic_thread_fence_acq();
567  } while (gen == 0 || gen != th->th_generation);
568 }
569 
570 void
571 getnanouptime(struct timespec *tsp)
572 {
573  struct timehands *th;
574  uint32_t gen;
575 
576  do {
577  th = timehands;
578  gen = atomic_load_acq_int(&th->th_generation);
579  bintime2timespec(&th->th_offset, tsp);
580  atomic_thread_fence_acq();
581  } while (gen == 0 || gen != th->th_generation);
582 }
583 
584 void
585 getmicrouptime(struct timeval *tvp)
586 {
587  struct timehands *th;
588  uint32_t gen;
589 
590  do {
591  th = timehands;
592  gen = atomic_load_acq_int(&th->th_generation);
593  bintime2timeval(&th->th_offset, tvp);
594  atomic_thread_fence_acq();
595  } while (gen == 0 || gen != th->th_generation);
596 }
597 
598 void
599 getbintime(struct bintime *bt)
600 {
601  struct timehands *th;
602  uint32_t gen;
603 
604  do {
605  th = timehands;
606  gen = atomic_load_acq_int(&th->th_generation);
607  *bt = th->th_bintime;
608  atomic_thread_fence_acq();
609  } while (gen == 0 || gen != th->th_generation);
610 }
611 
612 void
613 getnanotime(struct timespec *tsp)
614 {
615  struct timehands *th;
616  uint32_t gen;
617 
618  do {
619  th = timehands;
620  gen = atomic_load_acq_int(&th->th_generation);
621  *tsp = th->th_nanotime;
622  atomic_thread_fence_acq();
623  } while (gen == 0 || gen != th->th_generation);
624 }
625 
626 void
627 getmicrotime(struct timeval *tvp)
628 {
629  struct timehands *th;
630  uint32_t gen;
631 
632  do {
633  th = timehands;
634  gen = atomic_load_acq_int(&th->th_generation);
635  *tvp = th->th_microtime;
636  atomic_thread_fence_acq();
637  } while (gen == 0 || gen != th->th_generation);
638 }
639 #endif /* FFCLOCK */
640 
641 void
642 getboottime(struct timeval *boottime)
643 {
644  struct bintime boottimebin;
645 
646  getboottimebin(&boottimebin);
647  bintime2timeval(&boottimebin, boottime);
648 }
649 
650 void
651 getboottimebin(struct bintime *boottimebin)
652 {
653  struct timehands *th;
654  u_int gen;
655 
656  do {
657  th = timehands;
658  gen = atomic_load_acq_int(&th->th_generation);
659  *boottimebin = th->th_boottime;
660  atomic_thread_fence_acq();
661  } while (gen == 0 || gen != th->th_generation);
662 }
663 
664 #ifdef FFCLOCK
665 /*
666  * Support for feed-forward synchronization algorithms. This is heavily inspired
667  * by the timehands mechanism but kept independent from it. *_windup() functions
668  * have some connection to avoid accessing the timecounter hardware more than
669  * necessary.
670  */
671 
672 /* Feed-forward clock estimates kept updated by the synchronization daemon. */
674 struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */
675 uint32_t ffclock_status; /* Feed-forward clock status. */
676 int8_t ffclock_updated; /* New estimates are available. */
677 struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */
678 
679 struct fftimehands {
680  struct ffclock_estimate cest;
681  struct bintime tick_time;
682  struct bintime tick_time_lerp;
683  ffcounter tick_ffcount;
684  uint64_t period_lerp;
685  volatile uint8_t gen;
686  struct fftimehands *next;
687 };
688 
689 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
690 
691 static struct fftimehands ffth[10];
692 static struct fftimehands *volatile fftimehands = ffth;
693 
694 static void
695 ffclock_init(void)
696 {
697  struct fftimehands *cur;
698  struct fftimehands *last;
699 
700  memset(ffth, 0, sizeof(ffth));
701 
702  last = ffth + NUM_ELEMENTS(ffth) - 1;
703  for (cur = ffth; cur < last; cur++)
704  cur->next = cur + 1;
705  last->next = ffth;
706 
707  ffclock_updated = 0;
708  ffclock_status = FFCLOCK_STA_UNSYNC;
709  mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
710 }
711 
712 /*
713  * Reset the feed-forward clock estimates. Called from inittodr() to get things
714  * kick started and uses the timecounter nominal frequency as a first period
715  * estimate. Note: this function may be called several time just after boot.
716  * Note: this is the only function that sets the value of boot time for the
717  * monotonic (i.e. uptime) version of the feed-forward clock.
718  */
719 void
720 ffclock_reset_clock(struct timespec *ts)
721 {
722  struct timecounter *tc;
723  struct ffclock_estimate cest;
724 
725  tc = timehands->th_counter;
726  memset(&cest, 0, sizeof(struct ffclock_estimate));
727 
728  timespec2bintime(ts, &ffclock_boottime);
729  timespec2bintime(ts, &(cest.update_time));
730  ffclock_read_counter(&cest.update_ffcount);
731  cest.leapsec_next = 0;
732  cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
733  cest.errb_abs = 0;
734  cest.errb_rate = 0;
735  cest.status = FFCLOCK_STA_UNSYNC;
736  cest.leapsec_total = 0;
737  cest.leapsec = 0;
738 
739  mtx_lock(&ffclock_mtx);
740  bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
741  ffclock_updated = INT8_MAX;
742  mtx_unlock(&ffclock_mtx);
743 
744  printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
745  (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
746  (unsigned long)ts->tv_nsec);
747 }
748 
749 /*
750  * Sub-routine to convert a time interval measured in RAW counter units to time
751  * in seconds stored in bintime format.
752  * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
753  * larger than the max value of u_int (on 32 bit architecture). Loop to consume
754  * extra cycles.
755  */
756 static void
757 ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
758 {
759  struct bintime bt2;
760  ffcounter delta, delta_max;
761 
762  delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
763  bintime_clear(bt);
764  do {
765  if (ffdelta > delta_max)
766  delta = delta_max;
767  else
768  delta = ffdelta;
769  bt2.sec = 0;
770  bt2.frac = period;
771  bintime_mul(&bt2, (unsigned int)delta);
772  bintime_add(bt, &bt2);
773  ffdelta -= delta;
774  } while (ffdelta > 0);
775 }
776 
777 /*
778  * Update the fftimehands.
779  * Push the tick ffcount and time(s) forward based on current clock estimate.
780  * The conversion from ffcounter to bintime relies on the difference clock
781  * principle, whose accuracy relies on computing small time intervals. If a new
782  * clock estimate has been passed by the synchronisation daemon, make it
783  * current, and compute the linear interpolation for monotonic time if needed.
784  */
785 static void
786 ffclock_windup(unsigned int delta)
787 {
788  struct ffclock_estimate *cest;
789  struct fftimehands *ffth;
790  struct bintime bt, gap_lerp;
791  ffcounter ffdelta;
792  uint64_t frac;
793  unsigned int polling;
794  uint8_t forward_jump, ogen;
795 
796  /*
797  * Pick the next timehand, copy current ffclock estimates and move tick
798  * times and counter forward.
799  */
800  forward_jump = 0;
801  ffth = fftimehands->next;
802  ogen = ffth->gen;
803  ffth->gen = 0;
804  cest = &ffth->cest;
805  bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
806  ffdelta = (ffcounter)delta;
807  ffth->period_lerp = fftimehands->period_lerp;
808 
809  ffth->tick_time = fftimehands->tick_time;
810  ffclock_convert_delta(ffdelta, cest->period, &bt);
811  bintime_add(&ffth->tick_time, &bt);
812 
813  ffth->tick_time_lerp = fftimehands->tick_time_lerp;
814  ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
815  bintime_add(&ffth->tick_time_lerp, &bt);
816 
817  ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
818 
819  /*
820  * Assess the status of the clock, if the last update is too old, it is
821  * likely the synchronisation daemon is dead and the clock is free
822  * running.
823  */
824  if (ffclock_updated == 0) {
825  ffdelta = ffth->tick_ffcount - cest->update_ffcount;
826  ffclock_convert_delta(ffdelta, cest->period, &bt);
827  if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
828  ffclock_status |= FFCLOCK_STA_UNSYNC;
829  }
830 
831  /*
832  * If available, grab updated clock estimates and make them current.
833  * Recompute time at this tick using the updated estimates. The clock
834  * estimates passed the feed-forward synchronisation daemon may result
835  * in time conversion that is not monotonically increasing (just after
836  * the update). time_lerp is a particular linear interpolation over the
837  * synchronisation algo polling period that ensures monotonicity for the
838  * clock ids requesting it.
839  */
840  if (ffclock_updated > 0) {
841  bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
842  ffdelta = ffth->tick_ffcount - cest->update_ffcount;
843  ffth->tick_time = cest->update_time;
844  ffclock_convert_delta(ffdelta, cest->period, &bt);
845  bintime_add(&ffth->tick_time, &bt);
846 
847  /* ffclock_reset sets ffclock_updated to INT8_MAX */
848  if (ffclock_updated == INT8_MAX)
849  ffth->tick_time_lerp = ffth->tick_time;
850 
851  if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
852  forward_jump = 1;
853  else
854  forward_jump = 0;
855 
856  bintime_clear(&gap_lerp);
857  if (forward_jump) {
858  gap_lerp = ffth->tick_time;
859  bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
860  } else {
861  gap_lerp = ffth->tick_time_lerp;
862  bintime_sub(&gap_lerp, &ffth->tick_time);
863  }
864 
865  /*
866  * The reset from the RTC clock may be far from accurate, and
867  * reducing the gap between real time and interpolated time
868  * could take a very long time if the interpolated clock insists
869  * on strict monotonicity. The clock is reset under very strict
870  * conditions (kernel time is known to be wrong and
871  * synchronization daemon has been restarted recently.
872  * ffclock_boottime absorbs the jump to ensure boot time is
873  * correct and uptime functions stay consistent.
874  */
875  if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
876  ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
877  ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
878  if (forward_jump)
879  bintime_add(&ffclock_boottime, &gap_lerp);
880  else
881  bintime_sub(&ffclock_boottime, &gap_lerp);
882  ffth->tick_time_lerp = ffth->tick_time;
883  bintime_clear(&gap_lerp);
884  }
885 
886  ffclock_status = cest->status;
887  ffth->period_lerp = cest->period;
888 
889  /*
890  * Compute corrected period used for the linear interpolation of
891  * time. The rate of linear interpolation is capped to 5000PPM
892  * (5ms/s).
893  */
894  if (bintime_isset(&gap_lerp)) {
895  ffdelta = cest->update_ffcount;
896  ffdelta -= fftimehands->cest.update_ffcount;
897  ffclock_convert_delta(ffdelta, cest->period, &bt);
898  polling = bt.sec;
899  bt.sec = 0;
900  bt.frac = 5000000 * (uint64_t)18446744073LL;
901  bintime_mul(&bt, polling);
902  if (bintime_cmp(&gap_lerp, &bt, >))
903  gap_lerp = bt;
904 
905  /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
906  frac = 0;
907  if (gap_lerp.sec > 0) {
908  frac -= 1;
909  frac /= ffdelta / gap_lerp.sec;
910  }
911  frac += gap_lerp.frac / ffdelta;
912 
913  if (forward_jump)
914  ffth->period_lerp += frac;
915  else
916  ffth->period_lerp -= frac;
917  }
918 
919  ffclock_updated = 0;
920  }
921  if (++ogen == 0)
922  ogen = 1;
923  ffth->gen = ogen;
924  fftimehands = ffth;
925 }
926 
927 /*
928  * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
929  * the old and new hardware counter cannot be read simultaneously. tc_windup()
930  * does read the two counters 'back to back', but a few cycles are effectively
931  * lost, and not accumulated in tick_ffcount. This is a fairly radical
932  * operation for a feed-forward synchronization daemon, and it is its job to not
933  * pushing irrelevant data to the kernel. Because there is no locking here,
934  * simply force to ignore pending or next update to give daemon a chance to
935  * realize the counter has changed.
936  */
937 static void
938 ffclock_change_tc(struct timehands *th)
939 {
940  struct fftimehands *ffth;
941  struct ffclock_estimate *cest;
942  struct timecounter *tc;
943  uint8_t ogen;
944 
945  tc = th->th_counter;
946  ffth = fftimehands->next;
947  ogen = ffth->gen;
948  ffth->gen = 0;
949 
950  cest = &ffth->cest;
951  bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
952  cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
953  cest->errb_abs = 0;
954  cest->errb_rate = 0;
955  cest->status |= FFCLOCK_STA_UNSYNC;
956 
957  ffth->tick_ffcount = fftimehands->tick_ffcount;
958  ffth->tick_time_lerp = fftimehands->tick_time_lerp;
959  ffth->tick_time = fftimehands->tick_time;
960  ffth->period_lerp = cest->period;
961 
962  /* Do not lock but ignore next update from synchronization daemon. */
963  ffclock_updated--;
964 
965  if (++ogen == 0)
966  ogen = 1;
967  ffth->gen = ogen;
968  fftimehands = ffth;
969 }
970 
971 /*
972  * Retrieve feed-forward counter and time of last kernel tick.
973  */
974 void
975 ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
976 {
977  struct fftimehands *ffth;
978  uint8_t gen;
979 
980  /*
981  * No locking but check generation has not changed. Also need to make
982  * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
983  */
984  do {
985  ffth = fftimehands;
986  gen = ffth->gen;
987  if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
988  *bt = ffth->tick_time_lerp;
989  else
990  *bt = ffth->tick_time;
991  *ffcount = ffth->tick_ffcount;
992  } while (gen == 0 || gen != ffth->gen);
993 }
994 
995 /*
996  * Absolute clock conversion. Low level function to convert ffcounter to
997  * bintime. The ffcounter is converted using the current ffclock period estimate
998  * or the "interpolated period" to ensure monotonicity.
999  * NOTE: this conversion may have been deferred, and the clock updated since the
1000  * hardware counter has been read.
1001  */
1002 void
1003 ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
1004 {
1005  struct fftimehands *ffth;
1006  struct bintime bt2;
1007  ffcounter ffdelta;
1008  uint8_t gen;
1009 
1010  /*
1011  * No locking but check generation has not changed. Also need to make
1012  * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
1013  */
1014  do {
1015  ffth = fftimehands;
1016  gen = ffth->gen;
1017  if (ffcount > ffth->tick_ffcount)
1018  ffdelta = ffcount - ffth->tick_ffcount;
1019  else
1020  ffdelta = ffth->tick_ffcount - ffcount;
1021 
1022  if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
1023  *bt = ffth->tick_time_lerp;
1024  ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
1025  } else {
1026  *bt = ffth->tick_time;
1027  ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
1028  }
1029 
1030  if (ffcount > ffth->tick_ffcount)
1031  bintime_add(bt, &bt2);
1032  else
1033  bintime_sub(bt, &bt2);
1034  } while (gen == 0 || gen != ffth->gen);
1035 }
1036 
1037 /*
1038  * Difference clock conversion.
1039  * Low level function to Convert a time interval measured in RAW counter units
1040  * into bintime. The difference clock allows measuring small intervals much more
1041  * reliably than the absolute clock.
1042  */
1043 void
1044 ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
1045 {
1046  struct fftimehands *ffth;
1047  uint8_t gen;
1048 
1049  /* No locking but check generation has not changed. */
1050  do {
1051  ffth = fftimehands;
1052  gen = ffth->gen;
1053  ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
1054  } while (gen == 0 || gen != ffth->gen);
1055 }
1056 
1057 /*
1058  * Access to current ffcounter value.
1059  */
1060 void
1061 ffclock_read_counter(ffcounter *ffcount)
1062 {
1063  struct timehands *th;
1064  struct fftimehands *ffth;
1065  unsigned int gen, delta;
1066 
1067  /*
1068  * ffclock_windup() called from tc_windup(), safe to rely on
1069  * th->th_generation only, for correct delta and ffcounter.
1070  */
1071  do {
1072  th = timehands;
1073  gen = atomic_load_acq_int(&th->th_generation);
1074  ffth = fftimehands;
1075  delta = tc_delta(th);
1076  *ffcount = ffth->tick_ffcount;
1077  atomic_thread_fence_acq();
1078  } while (gen == 0 || gen != th->th_generation);
1079 
1080  *ffcount += delta;
1081 }
1082 
1083 void
1084 binuptime(struct bintime *bt)
1085 {
1086 
1087  binuptime_fromclock(bt, sysclock_active);
1088 }
1089 
1090 void
1091 nanouptime(struct timespec *tsp)
1092 {
1093 
1094  nanouptime_fromclock(tsp, sysclock_active);
1095 }
1096 
1097 void
1098 microuptime(struct timeval *tvp)
1099 {
1100 
1101  microuptime_fromclock(tvp, sysclock_active);
1102 }
1103 
1104 void
1105 bintime(struct bintime *bt)
1106 {
1107 
1108  bintime_fromclock(bt, sysclock_active);
1109 }
1110 
1111 void
1112 nanotime(struct timespec *tsp)
1113 {
1114 
1115  nanotime_fromclock(tsp, sysclock_active);
1116 }
1117 
1118 void
1119 microtime(struct timeval *tvp)
1120 {
1121 
1122  microtime_fromclock(tvp, sysclock_active);
1123 }
1124 
1125 void
1126 getbinuptime(struct bintime *bt)
1127 {
1128 
1129  getbinuptime_fromclock(bt, sysclock_active);
1130 }
1131 
1132 void
1133 getnanouptime(struct timespec *tsp)
1134 {
1135 
1136  getnanouptime_fromclock(tsp, sysclock_active);
1137 }
1138 
1139 void
1140 getmicrouptime(struct timeval *tvp)
1141 {
1142 
1143  getmicrouptime_fromclock(tvp, sysclock_active);
1144 }
1145 
1146 void
1147 getbintime(struct bintime *bt)
1148 {
1149 
1150  getbintime_fromclock(bt, sysclock_active);
1151 }
1152 
1153 void
1154 getnanotime(struct timespec *tsp)
1155 {
1156 
1157  getnanotime_fromclock(tsp, sysclock_active);
1158 }
1159 
1160 void
1161 getmicrotime(struct timeval *tvp)
1162 {
1163 
1164  getmicrouptime_fromclock(tvp, sysclock_active);
1165 }
1166 
1167 #endif /* FFCLOCK */
1168 
1169 #ifndef __rtems__
1170 /*
1171  * This is a clone of getnanotime and used for walltimestamps.
1172  * The dtrace_ prefix prevents fbt from creating probes for
1173  * it so walltimestamp can be safely used in all fbt probes.
1174  */
1175 void
1176 dtrace_getnanotime(struct timespec *tsp)
1177 {
1178  struct timehands *th;
1179  uint32_t gen;
1180 
1181  do {
1182  th = timehands;
1183  gen = atomic_load_acq_int(&th->th_generation);
1184  *tsp = th->th_nanotime;
1185  atomic_thread_fence_acq();
1186  } while (gen == 0 || gen != th->th_generation);
1187 }
1188 #endif /* __rtems__ */
1189 
1190 #ifdef FFCLOCK
1191 /*
1192  * System clock currently providing time to the system. Modifiable via sysctl
1193  * when the FFCLOCK option is defined.
1194  */
1195 int sysclock_active = SYSCLOCK_FBCK;
1196 #endif
1197 
1198 /* Internal NTP status and error estimates. */
1199 extern int time_status;
1200 extern long time_esterror;
1201 
1202 #ifndef __rtems__
1203 /*
1204  * Take a snapshot of sysclock data which can be used to compare system clocks
1205  * and generate timestamps after the fact.
1206  */
1207 void
1208 sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
1209 {
1210  struct fbclock_info *fbi;
1211  struct timehands *th;
1212  struct bintime bt;
1213  unsigned int delta, gen;
1214 #ifdef FFCLOCK
1215  ffcounter ffcount;
1216  struct fftimehands *ffth;
1217  struct ffclock_info *ffi;
1218  struct ffclock_estimate cest;
1219 
1220  ffi = &clock_snap->ff_info;
1221 #endif
1222 
1223  fbi = &clock_snap->fb_info;
1224  delta = 0;
1225 
1226  do {
1227  th = timehands;
1228  gen = atomic_load_acq_int(&th->th_generation);
1229  fbi->th_scale = th->th_scale;
1230  fbi->tick_time = th->th_offset;
1231 #ifdef FFCLOCK
1232  ffth = fftimehands;
1233  ffi->tick_time = ffth->tick_time_lerp;
1234  ffi->tick_time_lerp = ffth->tick_time_lerp;
1235  ffi->period = ffth->cest.period;
1236  ffi->period_lerp = ffth->period_lerp;
1237  clock_snap->ffcount = ffth->tick_ffcount;
1238  cest = ffth->cest;
1239 #endif
1240  if (!fast)
1241  delta = tc_delta(th);
1242  atomic_thread_fence_acq();
1243  } while (gen == 0 || gen != th->th_generation);
1244 
1245  clock_snap->delta = delta;
1246 #ifdef FFCLOCK
1247  clock_snap->sysclock_active = sysclock_active;
1248 #endif
1249 
1250  /* Record feedback clock status and error. */
1251  clock_snap->fb_info.status = time_status;
1252  /* XXX: Very crude estimate of feedback clock error. */
1253  bt.sec = time_esterror / 1000000;
1254  bt.frac = ((time_esterror - bt.sec) * 1000000) *
1255  (uint64_t)18446744073709ULL;
1256  clock_snap->fb_info.error = bt;
1257 
1258 #ifdef FFCLOCK
1259  if (!fast)
1260  clock_snap->ffcount += delta;
1261 
1262  /* Record feed-forward clock leap second adjustment. */
1263  ffi->leapsec_adjustment = cest.leapsec_total;
1264  if (clock_snap->ffcount > cest.leapsec_next)
1265  ffi->leapsec_adjustment -= cest.leapsec;
1266 
1267  /* Record feed-forward clock status and error. */
1268  clock_snap->ff_info.status = cest.status;
1269  ffcount = clock_snap->ffcount - cest.update_ffcount;
1270  ffclock_convert_delta(ffcount, cest.period, &bt);
1271  /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */
1272  bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL);
1273  /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */
1274  bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL);
1275  clock_snap->ff_info.error = bt;
1276 #endif
1277 }
1278 
1279 /*
1280  * Convert a sysclock snapshot into a struct bintime based on the specified
1281  * clock source and flags.
1282  */
1283 int
1284 sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt,
1285  int whichclock, uint32_t flags)
1286 {
1287  struct bintime boottimebin;
1288 #ifdef FFCLOCK
1289  struct bintime bt2;
1290  uint64_t period;
1291 #endif
1292 
1293  switch (whichclock) {
1294  case SYSCLOCK_FBCK:
1295  *bt = cs->fb_info.tick_time;
1296 
1297  /* If snapshot was created with !fast, delta will be >0. */
1298  if (cs->delta > 0)
1299  bintime_addx(bt, cs->fb_info.th_scale * cs->delta);
1300 
1301  if ((flags & FBCLOCK_UPTIME) == 0) {
1302  getboottimebin(&boottimebin);
1303  bintime_add(bt, &boottimebin);
1304  }
1305  break;
1306 #ifdef FFCLOCK
1307  case SYSCLOCK_FFWD:
1308  if (flags & FFCLOCK_LERP) {
1309  *bt = cs->ff_info.tick_time_lerp;
1310  period = cs->ff_info.period_lerp;
1311  } else {
1312  *bt = cs->ff_info.tick_time;
1313  period = cs->ff_info.period;
1314  }
1315 
1316  /* If snapshot was created with !fast, delta will be >0. */
1317  if (cs->delta > 0) {
1318  ffclock_convert_delta(cs->delta, period, &bt2);
1319  bintime_add(bt, &bt2);
1320  }
1321 
1322  /* Leap second adjustment. */
1323  if (flags & FFCLOCK_LEAPSEC)
1324  bt->sec -= cs->ff_info.leapsec_adjustment;
1325 
1326  /* Boot time adjustment, for uptime/monotonic clocks. */
1327  if (flags & FFCLOCK_UPTIME)
1328  bintime_sub(bt, &ffclock_boottime);
1329  break;
1330 #endif
1331  default:
1332  return (EINVAL);
1333  break;
1334  }
1335 
1336  return (0);
1337 }
1338 #endif /* __rtems__ */
1339 
1340 /*
1341  * Initialize a new timecounter and possibly use it.
1342  */
1343 void
1344 tc_init(struct timecounter *tc)
1345 {
1346 #ifndef __rtems__
1347  uint32_t u;
1348  struct sysctl_oid *tc_root;
1349 
1350  u = tc->tc_frequency / tc->tc_counter_mask;
1351  /* XXX: We need some margin here, 10% is a guess */
1352  u *= 11;
1353  u /= 10;
1354  if (u > hz && tc->tc_quality >= 0) {
1355  tc->tc_quality = -2000;
1356  if (bootverbose) {
1357  printf("Timecounter \"%s\" frequency %ju Hz",
1358  tc->tc_name, (uintmax_t)tc->tc_frequency);
1359  printf(" -- Insufficient hz, needs at least %u\n", u);
1360  }
1361  } else if (tc->tc_quality >= 0 || bootverbose) {
1362  printf("Timecounter \"%s\" frequency %ju Hz quality %d\n",
1363  tc->tc_name, (uintmax_t)tc->tc_frequency,
1364  tc->tc_quality);
1365  }
1366 
1367  tc->tc_next = timecounters;
1368  timecounters = tc;
1369  /*
1370  * Set up sysctl tree for this counter.
1371  */
1372  tc_root = SYSCTL_ADD_NODE_WITH_LABEL(NULL,
1373  SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name,
1374  CTLFLAG_RW, 0, "timecounter description", "timecounter");
1375  SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1376  "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0,
1377  "mask for implemented bits");
1378  SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1379  "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc),
1380  sysctl_kern_timecounter_get, "IU", "current timecounter value");
1381  SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1382  "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc),
1383  sysctl_kern_timecounter_freq, "QU", "timecounter frequency");
1384  SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1385  "quality", CTLFLAG_RD, &(tc->tc_quality), 0,
1386  "goodness of time counter");
1387  /*
1388  * Do not automatically switch if the current tc was specifically
1389  * chosen. Never automatically use a timecounter with negative quality.
1390  * Even though we run on the dummy counter, switching here may be
1391  * worse since this timecounter may not be monotonic.
1392  */
1393  if (tc_chosen)
1394  return;
1395  if (tc->tc_quality < 0)
1396  return;
1397 #endif /* __rtems__ */
1398  if (tc->tc_quality < timecounter->tc_quality)
1399  return;
1400  if (tc->tc_quality == timecounter->tc_quality &&
1401  tc->tc_frequency < timecounter->tc_frequency)
1402  return;
1403 #ifndef __rtems__
1404  (void)tc->tc_get_timecount(tc);
1405  (void)tc->tc_get_timecount(tc);
1406 #endif /* __rtems__ */
1407  timecounter = tc;
1408 #ifdef __rtems__
1409  tc_windup(NULL);
1410 #endif /* __rtems__ */
1411 }
1412 
1413 #ifndef __rtems__
1414 /* Report the frequency of the current timecounter. */
1415 uint64_t
1416 tc_getfrequency(void)
1417 {
1418 
1419  return (timehands->th_counter->tc_frequency);
1420 }
1421 
1422 static bool
1423 sleeping_on_old_rtc(struct thread *td)
1424 {
1425 
1426  /*
1427  * td_rtcgen is modified by curthread when it is running,
1428  * and by other threads in this function. By finding the thread
1429  * on a sleepqueue and holding the lock on the sleepqueue
1430  * chain, we guarantee that the thread is not running and that
1431  * modifying td_rtcgen is safe. Setting td_rtcgen to zero informs
1432  * the thread that it was woken due to a real-time clock adjustment.
1433  * (The declaration of td_rtcgen refers to this comment.)
1434  */
1435  if (td->td_rtcgen != 0 && td->td_rtcgen != rtc_generation) {
1436  td->td_rtcgen = 0;
1437  return (true);
1438  }
1439  return (false);
1440 }
1441 
1442 static struct mtx tc_setclock_mtx;
1443 MTX_SYSINIT(tc_setclock_init, &tc_setclock_mtx, "tcsetc", MTX_SPIN);
1444 #endif /* __rtems__ */
1445 
1446 /*
1447  * Step our concept of UTC. This is done by modifying our estimate of
1448  * when we booted.
1449  */
1450 void
1451 #ifndef __rtems__
1452 tc_setclock(struct timespec *ts)
1453 #else /* __rtems__ */
1454 _Timecounter_Set_clock(const struct bintime *_bt,
1455  ISR_lock_Context *lock_context)
1456 #endif /* __rtems__ */
1457 {
1458 #ifndef __rtems__
1459  struct timespec tbef, taft;
1460 #endif /* __rtems__ */
1461  struct bintime bt, bt2;
1462 
1463 #ifndef __rtems__
1464  timespec2bintime(ts, &bt);
1465  nanotime(&tbef);
1466  mtx_lock_spin(&tc_setclock_mtx);
1467  cpu_tick_calibrate(1);
1468 #else /* __rtems__ */
1469  bt = *_bt;
1470 #endif /* __rtems__ */
1471  binuptime(&bt2);
1472  bintime_sub(&bt, &bt2);
1473 
1474  /* XXX fiddle all the little crinkly bits around the fiords... */
1475 #ifndef __rtems__
1476  tc_windup(&bt);
1477  mtx_unlock_spin(&tc_setclock_mtx);
1478 
1479  /* Avoid rtc_generation == 0, since td_rtcgen == 0 is special. */
1480  atomic_add_rel_int(&rtc_generation, 2);
1481  sleepq_chains_remove_matching(sleeping_on_old_rtc);
1482  if (timestepwarnings) {
1483  nanotime(&taft);
1484  log(LOG_INFO,
1485  "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
1486  (intmax_t)tbef.tv_sec, tbef.tv_nsec,
1487  (intmax_t)taft.tv_sec, taft.tv_nsec,
1488  (intmax_t)ts->tv_sec, ts->tv_nsec);
1489  }
1490 #else /* __rtems__ */
1491  _Timecounter_Windup(&bt, lock_context);
1492 #endif /* __rtems__ */
1493 }
1494 
1495 /*
1496  * Initialize the next struct timehands in the ring and make
1497  * it the active timehands. Along the way we might switch to a different
1498  * timecounter and/or do seconds processing in NTP. Slightly magic.
1499  */
1500 static void
1501 tc_windup(struct bintime *new_boottimebin)
1502 #ifdef __rtems__
1503 {
1504  ISR_lock_Context lock_context;
1505 
1506  _Timecounter_Acquire(&lock_context);
1507  _Timecounter_Windup(new_boottimebin, &lock_context);
1508 }
1509 
1510 static void
1511 _Timecounter_Windup(struct bintime *new_boottimebin,
1512  ISR_lock_Context *lock_context)
1513 #endif /* __rtems__ */
1514 {
1515  struct bintime bt;
1516  struct timehands *th, *tho;
1517  uint64_t scale;
1518  uint32_t delta, ncount, ogen;
1519  int i;
1520  time_t t;
1521 
1522  /*
1523  * Make the next timehands a copy of the current one, but do
1524  * not overwrite the generation or next pointer. While we
1525  * update the contents, the generation must be zero. We need
1526  * to ensure that the zero generation is visible before the
1527  * data updates become visible, which requires release fence.
1528  * For similar reasons, re-reading of the generation after the
1529  * data is read should use acquire fence.
1530  */
1531  tho = timehands;
1532 #if defined(RTEMS_SMP)
1533  th = tho->th_next;
1534 #else
1535  th = tho;
1536 #endif
1537  ogen = th->th_generation;
1538  th->th_generation = 0;
1539  atomic_thread_fence_rel();
1540 #if defined(RTEMS_SMP)
1541  bcopy(tho, th, offsetof(struct timehands, th_generation));
1542 #endif
1543  if (new_boottimebin != NULL)
1544  th->th_boottime = *new_boottimebin;
1545 
1546  /*
1547  * Capture a timecounter delta on the current timecounter and if
1548  * changing timecounters, a counter value from the new timecounter.
1549  * Update the offset fields accordingly.
1550  */
1551  delta = tc_delta(th);
1552  if (th->th_counter != timecounter)
1553  ncount = timecounter->tc_get_timecount(timecounter);
1554  else
1555  ncount = 0;
1556 #ifdef FFCLOCK
1557  ffclock_windup(delta);
1558 #endif
1559  th->th_offset_count += delta;
1560  th->th_offset_count &= th->th_counter->tc_counter_mask;
1561  while (delta > th->th_counter->tc_frequency) {
1562  /* Eat complete unadjusted seconds. */
1563  delta -= th->th_counter->tc_frequency;
1564  th->th_offset.sec++;
1565  }
1566  if ((delta > th->th_counter->tc_frequency / 2) &&
1567  (th->th_scale * delta < ((uint64_t)1 << 63))) {
1568  /* The product th_scale * delta just barely overflows. */
1569  th->th_offset.sec++;
1570  }
1571  bintime_addx(&th->th_offset, th->th_scale * delta);
1572 
1573 #ifndef __rtems__
1574  /*
1575  * Hardware latching timecounters may not generate interrupts on
1576  * PPS events, so instead we poll them. There is a finite risk that
1577  * the hardware might capture a count which is later than the one we
1578  * got above, and therefore possibly in the next NTP second which might
1579  * have a different rate than the current NTP second. It doesn't
1580  * matter in practice.
1581  */
1582  if (tho->th_counter->tc_poll_pps)
1583  tho->th_counter->tc_poll_pps(tho->th_counter);
1584 #endif /* __rtems__ */
1585 
1586  /*
1587  * Deal with NTP second processing. The for loop normally
1588  * iterates at most once, but in extreme situations it might
1589  * keep NTP sane if timeouts are not run for several seconds.
1590  * At boot, the time step can be large when the TOD hardware
1591  * has been read, so on really large steps, we call
1592  * ntp_update_second only twice. We need to call it twice in
1593  * case we missed a leap second.
1594  */
1595  bt = th->th_offset;
1596  bintime_add(&bt, &th->th_boottime);
1597  i = bt.sec - tho->th_microtime.tv_sec;
1598  if (i > LARGE_STEP)
1599  i = 2;
1600  for (; i > 0; i--) {
1601  t = bt.sec;
1602  ntp_update_second(&th->th_adjustment, &bt.sec);
1603  if (bt.sec != t)
1604  th->th_boottime.sec += bt.sec - t;
1605  }
1606  /* Update the UTC timestamps used by the get*() functions. */
1607  th->th_bintime = bt;
1608  bintime2timeval(&bt, &th->th_microtime);
1609  bintime2timespec(&bt, &th->th_nanotime);
1610 
1611  /* Now is a good time to change timecounters. */
1612  if (th->th_counter != timecounter) {
1613 #ifndef __rtems__
1614 #ifndef __arm__
1615  if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0)
1616  cpu_disable_c2_sleep++;
1617  if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0)
1618  cpu_disable_c2_sleep--;
1619 #endif
1620 #endif /* __rtems__ */
1621  th->th_counter = timecounter;
1622  th->th_offset_count = ncount;
1623 #ifndef __rtems__
1624  tc_min_ticktock_freq = max(1, timecounter->tc_frequency /
1625  (((uint64_t)timecounter->tc_counter_mask + 1) / 3));
1626 #endif /* __rtems__ */
1627 #ifdef FFCLOCK
1628  ffclock_change_tc(th);
1629 #endif
1630  }
1631 
1632  /*-
1633  * Recalculate the scaling factor. We want the number of 1/2^64
1634  * fractions of a second per period of the hardware counter, taking
1635  * into account the th_adjustment factor which the NTP PLL/adjtime(2)
1636  * processing provides us with.
1637  *
1638  * The th_adjustment is nanoseconds per second with 32 bit binary
1639  * fraction and we want 64 bit binary fraction of second:
1640  *
1641  * x = a * 2^32 / 10^9 = a * 4.294967296
1642  *
1643  * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
1644  * we can only multiply by about 850 without overflowing, that
1645  * leaves no suitably precise fractions for multiply before divide.
1646  *
1647  * Divide before multiply with a fraction of 2199/512 results in a
1648  * systematic undercompensation of 10PPM of th_adjustment. On a
1649  * 5000PPM adjustment this is a 0.05PPM error. This is acceptable.
1650  *
1651  * We happily sacrifice the lowest of the 64 bits of our result
1652  * to the goddess of code clarity.
1653  *
1654  */
1655  scale = (uint64_t)1 << 63;
1656  scale += (th->th_adjustment / 1024) * 2199;
1657  scale /= th->th_counter->tc_frequency;
1658  th->th_scale = scale * 2;
1659 
1660  /*
1661  * Now that the struct timehands is again consistent, set the new
1662  * generation number, making sure to not make it zero.
1663  */
1664  if (++ogen == 0)
1665  ogen = 1;
1666  atomic_store_rel_int(&th->th_generation, ogen);
1667 
1668  /* Go live with the new struct timehands. */
1669 #ifdef FFCLOCK
1670  switch (sysclock_active) {
1671  case SYSCLOCK_FBCK:
1672 #endif
1673  time_second = th->th_microtime.tv_sec;
1674  time_uptime = th->th_offset.sec;
1675 #ifdef FFCLOCK
1676  break;
1677  case SYSCLOCK_FFWD:
1678  time_second = fftimehands->tick_time_lerp.sec;
1679  time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
1680  break;
1681  }
1682 #endif
1683 
1684 #if defined(RTEMS_SMP)
1685  timehands = th;
1686 #endif
1687 #ifndef __rtems__
1688  timekeep_push_vdso();
1689 #endif /* __rtems__ */
1690 #ifdef __rtems__
1691  _Timecounter_Release(lock_context);
1692 #endif /* __rtems__ */
1693 }
1694 
1695 #ifndef __rtems__
1696 /* Report or change the active timecounter hardware. */
1697 static int
1698 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
1699 {
1700  char newname[32];
1701  struct timecounter *newtc, *tc;
1702  int error;
1703 
1704  tc = timecounter;
1705  strlcpy(newname, tc->tc_name, sizeof(newname));
1706 
1707  error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req);
1708  if (error != 0 || req->newptr == NULL)
1709  return (error);
1710  /* Record that the tc in use now was specifically chosen. */
1711  tc_chosen = 1;
1712  if (strcmp(newname, tc->tc_name) == 0)
1713  return (0);
1714  for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
1715  if (strcmp(newname, newtc->tc_name) != 0)
1716  continue;
1717 
1718  /* Warm up new timecounter. */
1719  (void)newtc->tc_get_timecount(newtc);
1720  (void)newtc->tc_get_timecount(newtc);
1721 
1722  timecounter = newtc;
1723 
1724  /*
1725  * The vdso timehands update is deferred until the next
1726  * 'tc_windup()'.
1727  *
1728  * This is prudent given that 'timekeep_push_vdso()' does not
1729  * use any locking and that it can be called in hard interrupt
1730  * context via 'tc_windup()'.
1731  */
1732  return (0);
1733  }
1734  return (EINVAL);
1735 }
1736 
1737 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW,
1738  0, 0, sysctl_kern_timecounter_hardware, "A",
1739  "Timecounter hardware selected");
1740 
1741 
1742 /* Report the available timecounter hardware. */
1743 static int
1744 sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS)
1745 {
1746  struct sbuf sb;
1747  struct timecounter *tc;
1748  int error;
1749 
1750  sbuf_new_for_sysctl(&sb, NULL, 0, req);
1751  for (tc = timecounters; tc != NULL; tc = tc->tc_next) {
1752  if (tc != timecounters)
1753  sbuf_putc(&sb, ' ');
1754  sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality);
1755  }
1756  error = sbuf_finish(&sb);
1757  sbuf_delete(&sb);
1758  return (error);
1759 }
1760 
1761 SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
1762  0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected");
1763 #endif /* __rtems__ */
1764 
1765 #ifndef __rtems__
1766 /*
1767  * RFC 2783 PPS-API implementation.
1768  */
1769 
1770 /*
1771  * Return true if the driver is aware of the abi version extensions in the
1772  * pps_state structure, and it supports at least the given abi version number.
1773  */
1774 static inline int
1775 abi_aware(struct pps_state *pps, int vers)
1776 {
1777 
1778  return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers);
1779 }
1780 
1781 static int
1782 pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps)
1783 {
1784  int err, timo;
1785  pps_seq_t aseq, cseq;
1786  struct timeval tv;
1787 
1788  if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
1789  return (EINVAL);
1790 
1791  /*
1792  * If no timeout is requested, immediately return whatever values were
1793  * most recently captured. If timeout seconds is -1, that's a request
1794  * to block without a timeout. WITNESS won't let us sleep forever
1795  * without a lock (we really don't need a lock), so just repeatedly
1796  * sleep a long time.
1797  */
1798  if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) {
1799  if (fapi->timeout.tv_sec == -1)
1800  timo = 0x7fffffff;
1801  else {
1802  tv.tv_sec = fapi->timeout.tv_sec;
1803  tv.tv_usec = fapi->timeout.tv_nsec / 1000;
1804  timo = tvtohz(&tv);
1805  }
1806  aseq = pps->ppsinfo.assert_sequence;
1807  cseq = pps->ppsinfo.clear_sequence;
1808  while (aseq == pps->ppsinfo.assert_sequence &&
1809  cseq == pps->ppsinfo.clear_sequence) {
1810  if (abi_aware(pps, 1) && pps->driver_mtx != NULL) {
1811  if (pps->flags & PPSFLAG_MTX_SPIN) {
1812  err = msleep_spin(pps, pps->driver_mtx,
1813  "ppsfch", timo);
1814  } else {
1815  err = msleep(pps, pps->driver_mtx, PCATCH,
1816  "ppsfch", timo);
1817  }
1818  } else {
1819  err = tsleep(pps, PCATCH, "ppsfch", timo);
1820  }
1821  if (err == EWOULDBLOCK) {
1822  if (fapi->timeout.tv_sec == -1) {
1823  continue;
1824  } else {
1825  return (ETIMEDOUT);
1826  }
1827  } else if (err != 0) {
1828  return (err);
1829  }
1830  }
1831  }
1832 
1833  pps->ppsinfo.current_mode = pps->ppsparam.mode;
1834  fapi->pps_info_buf = pps->ppsinfo;
1835 
1836  return (0);
1837 }
1838 
1839 int
1840 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
1841 {
1842  pps_params_t *app;
1843  struct pps_fetch_args *fapi;
1844 #ifdef FFCLOCK
1845  struct pps_fetch_ffc_args *fapi_ffc;
1846 #endif
1847 #ifdef PPS_SYNC
1848  struct pps_kcbind_args *kapi;
1849 #endif
1850 
1851  KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl"));
1852  switch (cmd) {
1853  case PPS_IOC_CREATE:
1854  return (0);
1855  case PPS_IOC_DESTROY:
1856  return (0);
1857  case PPS_IOC_SETPARAMS:
1858  app = (pps_params_t *)data;
1859  if (app->mode & ~pps->ppscap)
1860  return (EINVAL);
1861 #ifdef FFCLOCK
1862  /* Ensure only a single clock is selected for ffc timestamp. */
1863  if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
1864  return (EINVAL);
1865 #endif
1866  pps->ppsparam = *app;
1867  return (0);
1868  case PPS_IOC_GETPARAMS:
1869  app = (pps_params_t *)data;
1870  *app = pps->ppsparam;
1871  app->api_version = PPS_API_VERS_1;
1872  return (0);
1873  case PPS_IOC_GETCAP:
1874  *(int*)data = pps->ppscap;
1875  return (0);
1876  case PPS_IOC_FETCH:
1877  fapi = (struct pps_fetch_args *)data;
1878  return (pps_fetch(fapi, pps));
1879 #ifdef FFCLOCK
1880  case PPS_IOC_FETCH_FFCOUNTER:
1881  fapi_ffc = (struct pps_fetch_ffc_args *)data;
1882  if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
1883  PPS_TSFMT_TSPEC)
1884  return (EINVAL);
1885  if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
1886  return (EOPNOTSUPP);
1887  pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
1888  fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
1889  /* Overwrite timestamps if feedback clock selected. */
1890  switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
1891  case PPS_TSCLK_FBCK:
1892  fapi_ffc->pps_info_buf_ffc.assert_timestamp =
1893  pps->ppsinfo.assert_timestamp;
1894  fapi_ffc->pps_info_buf_ffc.clear_timestamp =
1895  pps->ppsinfo.clear_timestamp;
1896  break;
1897  case PPS_TSCLK_FFWD:
1898  break;
1899  default:
1900  break;
1901  }
1902  return (0);
1903 #endif /* FFCLOCK */
1904  case PPS_IOC_KCBIND:
1905 #ifdef PPS_SYNC
1906  kapi = (struct pps_kcbind_args *)data;
1907  /* XXX Only root should be able to do this */
1908  if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
1909  return (EINVAL);
1910  if (kapi->kernel_consumer != PPS_KC_HARDPPS)
1911  return (EINVAL);
1912  if (kapi->edge & ~pps->ppscap)
1913  return (EINVAL);
1914  pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) |
1915  (pps->kcmode & KCMODE_ABIFLAG);
1916  return (0);
1917 #else
1918  return (EOPNOTSUPP);
1919 #endif
1920  default:
1921  return (ENOIOCTL);
1922  }
1923 }
1924 
1925 void
1926 pps_init(struct pps_state *pps)
1927 {
1928  pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT;
1929  if (pps->ppscap & PPS_CAPTUREASSERT)
1930  pps->ppscap |= PPS_OFFSETASSERT;
1931  if (pps->ppscap & PPS_CAPTURECLEAR)
1932  pps->ppscap |= PPS_OFFSETCLEAR;
1933 #ifdef FFCLOCK
1934  pps->ppscap |= PPS_TSCLK_MASK;
1935 #endif
1936  pps->kcmode &= ~KCMODE_ABIFLAG;
1937 }
1938 
1939 void
1940 pps_init_abi(struct pps_state *pps)
1941 {
1942 
1943  pps_init(pps);
1944  if (pps->driver_abi > 0) {
1945  pps->kcmode |= KCMODE_ABIFLAG;
1946  pps->kernel_abi = PPS_ABI_VERSION;
1947  }
1948 }
1949 
1950 void
1951 pps_capture(struct pps_state *pps)
1952 {
1953  struct timehands *th;
1954 
1955  KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
1956  th = timehands;
1957  pps->capgen = atomic_load_acq_int(&th->th_generation);
1958  pps->capth = th;
1959 #ifdef FFCLOCK
1960  pps->capffth = fftimehands;
1961 #endif
1962  pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
1963  atomic_thread_fence_acq();
1964  if (pps->capgen != th->th_generation)
1965  pps->capgen = 0;
1966 }
1967 
1968 void
1969 pps_event(struct pps_state *pps, int event)
1970 {
1971  struct bintime bt;
1972  struct timespec ts, *tsp, *osp;
1973  uint32_t tcount, *pcount;
1974  int foff;
1975  pps_seq_t *pseq;
1976 #ifdef FFCLOCK
1977  struct timespec *tsp_ffc;
1978  pps_seq_t *pseq_ffc;
1979  ffcounter *ffcount;
1980 #endif
1981 #ifdef PPS_SYNC
1982  int fhard;
1983 #endif
1984 
1985  KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
1986  /* Nothing to do if not currently set to capture this event type. */
1987  if ((event & pps->ppsparam.mode) == 0)
1988  return;
1989  /* If the timecounter was wound up underneath us, bail out. */
1990  if (pps->capgen == 0 || pps->capgen !=
1991  atomic_load_acq_int(&pps->capth->th_generation))
1992  return;
1993 
1994  /* Things would be easier with arrays. */
1995  if (event == PPS_CAPTUREASSERT) {
1996  tsp = &pps->ppsinfo.assert_timestamp;
1997  osp = &pps->ppsparam.assert_offset;
1998  foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
1999 #ifdef PPS_SYNC
2000  fhard = pps->kcmode & PPS_CAPTUREASSERT;
2001 #endif
2002  pcount = &pps->ppscount[0];
2003  pseq = &pps->ppsinfo.assert_sequence;
2004 #ifdef FFCLOCK
2005  ffcount = &pps->ppsinfo_ffc.assert_ffcount;
2006  tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
2007  pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
2008 #endif
2009  } else {
2010  tsp = &pps->ppsinfo.clear_timestamp;
2011  osp = &pps->ppsparam.clear_offset;
2012  foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
2013 #ifdef PPS_SYNC
2014  fhard = pps->kcmode & PPS_CAPTURECLEAR;
2015 #endif
2016  pcount = &pps->ppscount[1];
2017  pseq = &pps->ppsinfo.clear_sequence;
2018 #ifdef FFCLOCK
2019  ffcount = &pps->ppsinfo_ffc.clear_ffcount;
2020  tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
2021  pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
2022 #endif
2023  }
2024 
2025  /*
2026  * If the timecounter changed, we cannot compare the count values, so
2027  * we have to drop the rest of the PPS-stuff until the next event.
2028  */
2029  if (pps->ppstc != pps->capth->th_counter) {
2030  pps->ppstc = pps->capth->th_counter;
2031  *pcount = pps->capcount;
2032  pps->ppscount[2] = pps->capcount;
2033  return;
2034  }
2035 
2036  /* Convert the count to a timespec. */
2037  tcount = pps->capcount - pps->capth->th_offset_count;
2038  tcount &= pps->capth->th_counter->tc_counter_mask;
2039  bt = pps->capth->th_bintime;
2040  bintime_addx(&bt, pps->capth->th_scale * tcount);
2041  bintime2timespec(&bt, &ts);
2042 
2043  /* If the timecounter was wound up underneath us, bail out. */
2044  atomic_thread_fence_acq();
2045  if (pps->capgen != pps->capth->th_generation)
2046  return;
2047 
2048  *pcount = pps->capcount;
2049  (*pseq)++;
2050  *tsp = ts;
2051 
2052  if (foff) {
2053  timespecadd(tsp, osp, tsp);
2054  if (tsp->tv_nsec < 0) {
2055  tsp->tv_nsec += 1000000000;
2056  tsp->tv_sec -= 1;
2057  }
2058  }
2059 
2060 #ifdef FFCLOCK
2061  *ffcount = pps->capffth->tick_ffcount + tcount;
2062  bt = pps->capffth->tick_time;
2063  ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
2064  bintime_add(&bt, &pps->capffth->tick_time);
2065  bintime2timespec(&bt, &ts);
2066  (*pseq_ffc)++;
2067  *tsp_ffc = ts;
2068 #endif
2069 
2070 #ifdef PPS_SYNC
2071  if (fhard) {
2072  uint64_t scale;
2073 
2074  /*
2075  * Feed the NTP PLL/FLL.
2076  * The FLL wants to know how many (hardware) nanoseconds
2077  * elapsed since the previous event.
2078  */
2079  tcount = pps->capcount - pps->ppscount[2];
2080  pps->ppscount[2] = pps->capcount;
2081  tcount &= pps->capth->th_counter->tc_counter_mask;
2082  scale = (uint64_t)1 << 63;
2083  scale /= pps->capth->th_counter->tc_frequency;
2084  scale *= 2;
2085  bt.sec = 0;
2086  bt.frac = 0;
2087  bintime_addx(&bt, scale * tcount);
2088  bintime2timespec(&bt, &ts);
2089  hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
2090  }
2091 #endif
2092 
2093  /* Wakeup anyone sleeping in pps_fetch(). */
2094  wakeup(pps);
2095 }
2096 #else /* __rtems__ */
2097 /* FIXME: https://devel.rtems.org/ticket/2349 */
2098 #endif /* __rtems__ */
2099 
2100 /*
2101  * Timecounters need to be updated every so often to prevent the hardware
2102  * counter from overflowing. Updating also recalculates the cached values
2103  * used by the get*() family of functions, so their precision depends on
2104  * the update frequency.
2105  */
2106 
2107 #ifndef __rtems__
2108 static int tc_tick;
2109 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0,
2110  "Approximate number of hardclock ticks in a millisecond");
2111 #endif /* __rtems__ */
2112 
2113 #ifndef __rtems__
2114 void
2115 tc_ticktock(int cnt)
2116 {
2117  static int count;
2118 
2119  if (mtx_trylock_spin(&tc_setclock_mtx)) {
2120  count += cnt;
2121  if (count >= tc_tick) {
2122  count = 0;
2123  tc_windup(NULL);
2124  }
2125  mtx_unlock_spin(&tc_setclock_mtx);
2126  }
2127 }
2128 #else /* __rtems__ */
2129 void
2131 {
2132  Per_CPU_Control *cpu_self = _Per_CPU_Get();
2133 
2134  if (_Per_CPU_Is_boot_processor(cpu_self)) {
2135  tc_windup(NULL);
2136  }
2137 
2138  _Watchdog_Tick(cpu_self);
2139 }
2140 
2141 void
2142 _Timecounter_Tick_simple(uint32_t delta, uint32_t offset,
2143  ISR_lock_Context *lock_context)
2144 {
2145  struct bintime bt;
2146  struct timehands *th;
2147  uint32_t ogen;
2148 
2149  th = timehands;
2150  ogen = th->th_generation;
2151  th->th_offset_count = offset;
2152  bintime_addx(&th->th_offset, th->th_scale * delta);
2153 
2154  bt = th->th_offset;
2155  bintime_add(&bt, &th->th_boottime);
2156  /* Update the UTC timestamps used by the get*() functions. */
2157  th->th_bintime = bt;
2158  bintime2timeval(&bt, &th->th_microtime);
2159  bintime2timespec(&bt, &th->th_nanotime);
2160 
2161  /*
2162  * Now that the struct timehands is again consistent, set the new
2163  * generation number, making sure to not make it zero.
2164  */
2165  if (++ogen == 0)
2166  ogen = 1;
2167  th->th_generation = ogen;
2168 
2169  /* Go live with the new struct timehands. */
2170  time_second = th->th_microtime.tv_sec;
2171  time_uptime = th->th_offset.sec;
2172 
2173  _Timecounter_Release(lock_context);
2174 
2175  _Watchdog_Tick(_Per_CPU_Get_snapshot());
2176 }
2177 #endif /* __rtems__ */
2178 
2179 #ifndef __rtems__
2180 static void __inline
2181 tc_adjprecision(void)
2182 {
2183  int t;
2184 
2185  if (tc_timepercentage > 0) {
2186  t = (99 + tc_timepercentage) / tc_timepercentage;
2187  tc_precexp = fls(t + (t >> 1)) - 1;
2188  FREQ2BT(hz / tc_tick, &bt_timethreshold);
2189  FREQ2BT(hz, &bt_tickthreshold);
2190  bintime_shift(&bt_timethreshold, tc_precexp);
2191  bintime_shift(&bt_tickthreshold, tc_precexp);
2192  } else {
2193  tc_precexp = 31;
2194  bt_timethreshold.sec = INT_MAX;
2195  bt_timethreshold.frac = ~(uint64_t)0;
2196  bt_tickthreshold = bt_timethreshold;
2197  }
2198  sbt_timethreshold = bttosbt(bt_timethreshold);
2199  sbt_tickthreshold = bttosbt(bt_tickthreshold);
2200 }
2201 #endif /* __rtems__ */
2202 
2203 #ifndef __rtems__
2204 static int
2205 sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS)
2206 {
2207  int error, val;
2208 
2209  val = tc_timepercentage;
2210  error = sysctl_handle_int(oidp, &val, 0, req);
2211  if (error != 0 || req->newptr == NULL)
2212  return (error);
2213  tc_timepercentage = val;
2214  if (cold)
2215  goto done;
2216  tc_adjprecision();
2217 done:
2218  return (0);
2219 }
2220 
2221 static void
2222 inittimecounter(void *dummy)
2223 {
2224  u_int p;
2225  int tick_rate;
2226 
2227  /*
2228  * Set the initial timeout to
2229  * max(1, <approx. number of hardclock ticks in a millisecond>).
2230  * People should probably not use the sysctl to set the timeout
2231  * to smaller than its initial value, since that value is the
2232  * smallest reasonable one. If they want better timestamps they
2233  * should use the non-"get"* functions.
2234  */
2235  if (hz > 1000)
2236  tc_tick = (hz + 500) / 1000;
2237  else
2238  tc_tick = 1;
2239  tc_adjprecision();
2240  FREQ2BT(hz, &tick_bt);
2241  tick_sbt = bttosbt(tick_bt);
2242  tick_rate = hz / tc_tick;
2243  FREQ2BT(tick_rate, &tc_tick_bt);
2244  tc_tick_sbt = bttosbt(tc_tick_bt);
2245  p = (tc_tick * 1000000) / hz;
2246  printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
2247 
2248 #ifdef FFCLOCK
2249  ffclock_init();
2250 #endif
2251  /* warm up new timecounter (again) and get rolling. */
2252  (void)timecounter->tc_get_timecount(timecounter);
2253  (void)timecounter->tc_get_timecount(timecounter);
2254  mtx_lock_spin(&tc_setclock_mtx);
2255  tc_windup(NULL);
2256  mtx_unlock_spin(&tc_setclock_mtx);
2257 }
2258 
2259 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
2260 
2261 /* Cpu tick handling -------------------------------------------------*/
2262 
2263 static int cpu_tick_variable;
2264 static uint64_t cpu_tick_frequency;
2265 
2266 static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base);
2267 static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last);
2268 
2269 static uint64_t
2270 tc_cpu_ticks(void)
2271 {
2272  struct timecounter *tc;
2273  uint64_t res, *base;
2274  unsigned u, *last;
2275 
2276  critical_enter();
2277  base = DPCPU_PTR(tc_cpu_ticks_base);
2278  last = DPCPU_PTR(tc_cpu_ticks_last);
2279  tc = timehands->th_counter;
2280  u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
2281  if (u < *last)
2282  *base += (uint64_t)tc->tc_counter_mask + 1;
2283  *last = u;
2284  res = u + *base;
2285  critical_exit();
2286  return (res);
2287 }
2288 
2289 void
2290 cpu_tick_calibration(void)
2291 {
2292  static time_t last_calib;
2293 
2294  if (time_uptime != last_calib && !(time_uptime & 0xf)) {
2295  cpu_tick_calibrate(0);
2296  last_calib = time_uptime;
2297  }
2298 }
2299 
2300 /*
2301  * This function gets called every 16 seconds on only one designated
2302  * CPU in the system from hardclock() via cpu_tick_calibration()().
2303  *
2304  * Whenever the real time clock is stepped we get called with reset=1
2305  * to make sure we handle suspend/resume and similar events correctly.
2306  */
2307 
2308 static void
2309 cpu_tick_calibrate(int reset)
2310 {
2311  static uint64_t c_last;
2312  uint64_t c_this, c_delta;
2313  static struct bintime t_last;
2314  struct bintime t_this, t_delta;
2315  uint32_t divi;
2316 
2317  if (reset) {
2318  /* The clock was stepped, abort & reset */
2319  t_last.sec = 0;
2320  return;
2321  }
2322 
2323  /* we don't calibrate fixed rate cputicks */
2324  if (!cpu_tick_variable)
2325  return;
2326 
2327  getbinuptime(&t_this);
2328  c_this = cpu_ticks();
2329  if (t_last.sec != 0) {
2330  c_delta = c_this - c_last;
2331  t_delta = t_this;
2332  bintime_sub(&t_delta, &t_last);
2333  /*
2334  * Headroom:
2335  * 2^(64-20) / 16[s] =
2336  * 2^(44) / 16[s] =
2337  * 17.592.186.044.416 / 16 =
2338  * 1.099.511.627.776 [Hz]
2339  */
2340  divi = t_delta.sec << 20;
2341  divi |= t_delta.frac >> (64 - 20);
2342  c_delta <<= 20;
2343  c_delta /= divi;
2344  if (c_delta > cpu_tick_frequency) {
2345  if (0 && bootverbose)
2346  printf("cpu_tick increased to %ju Hz\n",
2347  c_delta);
2348  cpu_tick_frequency = c_delta;
2349  }
2350  }
2351  c_last = c_this;
2352  t_last = t_this;
2353 }
2354 
2355 void
2356 set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var)
2357 {
2358 
2359  if (func == NULL) {
2360  cpu_ticks = tc_cpu_ticks;
2361  } else {
2362  cpu_tick_frequency = freq;
2363  cpu_tick_variable = var;
2364  cpu_ticks = func;
2365  }
2366 }
2367 
2368 uint64_t
2369 cpu_tickrate(void)
2370 {
2371 
2372  if (cpu_ticks == tc_cpu_ticks)
2373  return (tc_getfrequency());
2374  return (cpu_tick_frequency);
2375 }
2376 
2377 /*
2378  * We need to be slightly careful converting cputicks to microseconds.
2379  * There is plenty of margin in 64 bits of microseconds (half a million
2380  * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply
2381  * before divide conversion (to retain precision) we find that the
2382  * margin shrinks to 1.5 hours (one millionth of 146y).
2383  * With a three prong approach we never lose significant bits, no
2384  * matter what the cputick rate and length of timeinterval is.
2385  */
2386 
2387 uint64_t
2388 cputick2usec(uint64_t tick)
2389 {
2390 
2391  if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */
2392  return (tick / (cpu_tickrate() / 1000000LL));
2393  else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */
2394  return ((tick * 1000LL) / (cpu_tickrate() / 1000LL));
2395  else
2396  return ((tick * 1000000LL) / cpu_tickrate());
2397 }
2398 
2399 cpu_tick_f *cpu_ticks = tc_cpu_ticks;
2400 #endif /* __rtems__ */
2401 
2402 #ifndef __rtems__
2403 static int vdso_th_enable = 1;
2404 static int
2405 sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
2406 {
2407  int old_vdso_th_enable, error;
2408 
2409  old_vdso_th_enable = vdso_th_enable;
2410  error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
2411  if (error != 0)
2412  return (error);
2413  vdso_th_enable = old_vdso_th_enable;
2414  return (0);
2415 }
2416 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
2417  CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
2418  NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
2419 
2420 uint32_t
2421 tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
2422 {
2423  struct timehands *th;
2424  uint32_t enabled;
2425 
2426  th = timehands;
2427  vdso_th->th_scale = th->th_scale;
2428  vdso_th->th_offset_count = th->th_offset_count;
2429  vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
2430  vdso_th->th_offset = th->th_offset;
2431  vdso_th->th_boottime = th->th_boottime;
2432  if (th->th_counter->tc_fill_vdso_timehands != NULL) {
2433  enabled = th->th_counter->tc_fill_vdso_timehands(vdso_th,
2434  th->th_counter);
2435  } else
2436  enabled = 0;
2437  if (!vdso_th_enable)
2438  enabled = 0;
2439  return (enabled);
2440 }
2441 #endif /* __rtems__ */
2442 
2443 #ifdef COMPAT_FREEBSD32
2444 uint32_t
2445 tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
2446 {
2447  struct timehands *th;
2448  uint32_t enabled;
2449 
2450  th = timehands;
2451  *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
2452  vdso_th32->th_offset_count = th->th_offset_count;
2453  vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
2454  vdso_th32->th_offset.sec = th->th_offset.sec;
2455  *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
2456  vdso_th32->th_boottime.sec = th->th_boottime.sec;
2457  *(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac;
2458  if (th->th_counter->tc_fill_vdso_timehands32 != NULL) {
2459  enabled = th->th_counter->tc_fill_vdso_timehands32(vdso_th32,
2460  th->th_counter);
2461  } else
2462  enabled = 0;
2463  if (!vdso_th_enable)
2464  enabled = 0;
2465  return (enabled);
2466 }
2467 #endif
void _Timecounter_Tick_simple(uint32_t delta, uint32_t offset, ISR_lock_Context *lock_context)
Performs a simple timecounter tick.
Definition: kern_tc.c:2142
Inlined Routines in the Watchdog Handler.
#define _Timecounter_Release(lock_context)
Releases the timecounter lock.
Definition: timecounter.h:211
#define _Timecounter_Acquire(lock_context)
Lock to protect the timecounter mechanic.
Definition: timecounter.h:201
SuperCore SMP Support API.
Per CPU Core Structure.
Definition: percpu.h:347
void _Timecounter_Tick(void)
Performs a timecounter tick.
Definition: kern_tc.c:2130
Timecounter Implementation.
Atomic Operations API.
void _Timecounter_Set_clock(const struct bintime *_bt, ISR_lock_Context *lock_context)
Sets the timecounter clock to the given value.
Definition: kern_tc.c:1454
Time of Day Handler API.
This header file defines the RTEMS Classic API.
#define ISR_LOCK_DEFINE(_qualifier, _designator, _name)
Defines an ISR lock variable.
Definition: isrlock.h:118
Local ISR lock context for acquire and release pairs.
Definition: isrlock.h:65
sbintime_t _Timecounter_Sbinuptime(void)
Returns the uptime in the sbintime_t format.
Definition: kern_tc.c:487
#define TOD_SECONDS_1970_THROUGH_1988
Definition: todimpl.h:111
void _Watchdog_Tick(struct Per_CPU_Control *cpu)
Performs a watchdog tick.
Definition: watchdogtick.c:54