timecounter: Synchronize with FreeBSD

When updating/accessing the timehands, barriers are needed to ensure
that:
- th_generation update is visible after the parameters update is
  visible;
- the read of parameters is not reordered before initial read of
  th_generation.

On UP kernels, compiler barriers are enough.  For SMP machines, CPU
barriers must be used too, as was confirmed by submitter by testing on
the Freescale T4240 platform with 24 PowerPC processors.

Submitted by:	Sebastian Huber <sebastian.huber@embedded-brains.de>
MFC after:	1 week

1 files changed, 82 insertions, 42 deletions

diff --git a/cpukit/score/src/kern_tc.c b/cpukit/score/src/kern_tc.c
index 654703dd4b..f6a1136d31 100644
--- a/cpukit/score/src/kern_tc.c
+++ b/cpukit/score/src/kern_tc.c
@@ -36,7 +36,7 @@
 #include <rtems/score/watchdogimpl.h>
 #endif /* __rtems__ */
 #include <sys/cdefs.h>
-__FBSDID("$FreeBSD r282424 2015-05-04T17:59:39Z$");
+__FBSDID("$FreeBSD r284178 2015-06-09T11:49:56Z$");
 
 #include "opt_compat.h"
 #include "opt_ntp.h"
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD r282424 2015-05-04T17:59:39Z$");
 #include <sys/timex.h>
 #ifndef __rtems__
 #include <sys/vdso.h>
+#include <machine/atomic.h>
 #endif /* __rtems__ */
 #ifdef __rtems__
 #include <limits.h>
@@ -116,7 +117,11 @@ struct timehands {
 	struct timeval		th_microtime;
 	struct timespec		th_nanotime;
 	/* Fields not to be copied in tc_windup start with th_generation. */
-	volatile uint32_t	th_generation;
+#ifndef __rtems__
+	u_int			th_generation;
+#else /* __rtems__ */
+	Atomic_Ulong		th_generation;
+#endif /* __rtems__ */
 	struct timehands	*th_next;
 };
 
@@ -248,6 +253,41 @@ tc_delta(struct timehands *th)
 	    tc->tc_counter_mask);
 }
 
+static u_int
+tc_getgen(struct timehands *th)
+{
+
+#ifndef __rtems__
+#ifdef SMP
+	return (atomic_load_acq_int(&th->th_generation));
+#else
+	u_int gen;
+
+	gen = th->th_generation;
+	__compiler_membar();
+	return (gen);
+#endif
+#else /* __rtems__ */
+	return (_Atomic_Load_ulong(&th->th_generation, ATOMIC_ORDER_ACQUIRE));
+#endif /* __rtems__ */
+}
+
+static void
+tc_setgen(struct timehands *th, u_int newgen)
+{
+
+#ifndef __rtems__
+#ifdef SMP
+	atomic_store_rel_int(&th->th_generation, newgen);
+#else
+	__compiler_membar();
+	th->th_generation = newgen;
+#endif
+#else /* __rtems__ */
+	_Atomic_Store_ulong(&th->th_generation, newgen, ATOMIC_ORDER_RELEASE);
+#endif /* __rtems__ */
+}
+
 /*
  * Functions for reading the time.  We have to loop until we are sure that
  * the timehands that we operated on was not updated under our feet.  See
@@ -263,10 +303,10 @@ fbclock_binuptime(struct bintime *bt)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*bt = th->th_offset;
 		bintime_addx(bt, th->th_scale * tc_delta(th));
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -321,9 +361,9 @@ fbclock_getbinuptime(struct bintime *bt)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*bt = th->th_offset;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -334,9 +374,9 @@ fbclock_getnanouptime(struct timespec *tsp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		bintime2timespec(&th->th_offset, tsp);
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -347,9 +387,9 @@ fbclock_getmicrouptime(struct timeval *tvp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		bintime2timeval(&th->th_offset, tvp);
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -360,9 +400,9 @@ fbclock_getbintime(struct bintime *bt)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*bt = th->th_offset;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 	bintime_add(bt, &boottimebin);
 }
 
@@ -374,9 +414,9 @@ fbclock_getnanotime(struct timespec *tsp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*tsp = th->th_nanotime;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -387,9 +427,9 @@ fbclock_getmicrotime(struct timeval *tvp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*tvp = th->th_microtime;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 #else /* !FFCLOCK */
 void
@@ -400,10 +440,10 @@ binuptime(struct bintime *bt)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*bt = th->th_offset;
 		bintime_addx(bt, th->th_scale * tc_delta(th));
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -458,9 +498,9 @@ getbinuptime(struct bintime *bt)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*bt = th->th_offset;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -471,9 +511,9 @@ getnanouptime(struct timespec *tsp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		bintime2timespec(&th->th_offset, tsp);
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -484,9 +524,9 @@ getmicrouptime(struct timeval *tvp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		bintime2timeval(&th->th_offset, tvp);
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -497,9 +537,9 @@ getbintime(struct bintime *bt)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*bt = th->th_offset;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 	bintime_add(bt, &boottimebin);
 }
 
@@ -511,9 +551,9 @@ getnanotime(struct timespec *tsp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*tsp = th->th_nanotime;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 
 void
@@ -524,9 +564,9 @@ getmicrotime(struct timeval *tvp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*tvp = th->th_microtime;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 #endif /* FFCLOCK */
 
@@ -939,11 +979,11 @@ ffclock_read_counter(ffcounter *ffcount)
 	 */
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		ffth = fftimehands;
 		delta = tc_delta(th);
 		*ffcount = ffth->tick_ffcount;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 
 	*ffcount += delta;
 }
@@ -1048,9 +1088,9 @@ dtrace_getnanotime(struct timespec *tsp)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		*tsp = th->th_nanotime;
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 }
 #endif /* __rtems__ */
 
@@ -1092,7 +1132,7 @@ sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
 
 	do {
 		th = timehands;
-		gen = th->th_generation;
+		gen = tc_getgen(th);
 		fbi->th_scale = th->th_scale;
 		fbi->tick_time = th->th_offset;
 #ifdef FFCLOCK
@@ -1106,7 +1146,7 @@ sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
 #endif
 		if (!fast)
 			delta = tc_delta(th);
-	} while (gen == 0 || gen != th->th_generation);
+	} while (gen == 0 || gen != tc_getgen(th));
 
 	clock_snap->delta = delta;
 #ifdef FFCLOCK
@@ -1354,7 +1394,7 @@ tc_windup(void)
 	tho = timehands;
 	th = tho->th_next;
 	ogen = th->th_generation;
-	th->th_generation = 0;
+	tc_setgen(th, 0);
 	bcopy(tho, th, offsetof(struct timehands, th_generation));
 
 	/*
@@ -1475,7 +1515,7 @@ tc_windup(void)
 	 */
 	if (++ogen == 0)
 		ogen = 1;
-	th->th_generation = ogen;
+	tc_setgen(th, ogen);
 
 	/* Go live with the new struct timehands. */
 #ifdef FFCLOCK
@@ -1757,13 +1797,13 @@ pps_capture(struct pps_state *pps)
 
 	KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
 	th = timehands;
-	pps->capgen = th->th_generation;
+	pps->capgen = tc_getgen(th);
 	pps->capth = th;
 #ifdef FFCLOCK
 	pps->capffth = fftimehands;
 #endif
 	pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
-	if (pps->capgen != th->th_generation)
+	if (pps->capgen != tc_getgen(th))
 		pps->capgen = 0;
 }
 
@@ -1783,7 +1823,7 @@ pps_event(struct pps_state *pps, int event)
 
 	KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
 	/* If the timecounter was wound up underneath us, bail out. */
-	if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation)
+	if (pps->capgen == 0 || pps->capgen != tc_getgen(pps->capth))
 		return;
 
 	/* Things would be easier with arrays. */
@@ -1833,7 +1873,7 @@ pps_event(struct pps_state *pps, int event)
 	bintime2timespec(&bt, &ts);
 
 	/* If the timecounter was wound up underneath us, bail out. */
-	if (pps->capgen != pps->capth->th_generation)
+	if (pps->capgen != tc_getgen(pps->capth))
 		return;
 
 	*pcount = pps->capcount;