diff options
Diffstat (limited to 'cpukit/score')
-rw-r--r-- | cpukit/score/cpu/sparc/cpu_asm.S | 11 | ||||
-rw-r--r-- | cpukit/score/cpu/sparc/sparc-access.S | 4 | ||||
-rw-r--r-- | cpukit/score/cpu/sparc/sparc-counter-asm.S | 3 | ||||
-rw-r--r-- | cpukit/score/cpu/sparc/syscall.S | 3 | ||||
-rw-r--r-- | cpukit/score/src/corebarrierrelease.c | 3 | ||||
-rw-r--r-- | cpukit/score/src/corebarrierwait.c | 32 | ||||
-rw-r--r-- | cpukit/score/src/heapallocate.c | 9 | ||||
-rw-r--r-- | cpukit/score/src/kern_tc.c | 676 | ||||
-rw-r--r-- | cpukit/score/src/objectactivecount.c | 18 | ||||
-rw-r--r-- | cpukit/score/src/objectfree.c | 12 | ||||
-rw-r--r-- | cpukit/score/src/stackallocatorforidle.c | 59 | ||||
-rw-r--r-- | cpukit/score/src/threadcreateidle.c | 12 | ||||
-rw-r--r-- | cpukit/score/src/threadqops.c | 27 |
13 files changed, 577 insertions, 292 deletions
diff --git a/cpukit/score/cpu/sparc/cpu_asm.S b/cpukit/score/cpu/sparc/cpu_asm.S index d5afd5f7b0..1251faa2f7 100644 --- a/cpukit/score/cpu/sparc/cpu_asm.S +++ b/cpukit/score/cpu/sparc/cpu_asm.S @@ -523,8 +523,9 @@ dont_do_the_window: * context. */ andcc %l0, %l5, %g0 - bne,a dont_switch_stacks - st %fsr, [%g6 + SPARC_PER_CPU_FSR_OFFSET] + beq dont_switch_stacks + nop + st %fsr, [%g6 + SPARC_PER_CPU_FSR_OFFSET] #endif dont_switch_stacks: @@ -683,11 +684,13 @@ isr_dispatch: cmp %l6, %l7 bne,a .Ldisable_fp andn %l0, %l5, %l0 + st %g0, [%g6 + SPARC_PER_CPU_FP_OWNER_OFFSET] ba .Lthread_dispatch_done - st %g0, [%g6 + SPARC_PER_CPU_FP_OWNER_OFFSET] + nop .Ldisable_fp: + st %l0, [%fp + ISF_PSR_OFFSET] ba .Lthread_dispatch_done - st %l0, [%fp + ISF_PSR_OFFSET] + nop .Lnon_fp_thread_dispatch: #elif defined(SPARC_USE_SYNCHRONOUS_FP_SWITCH) /* Test if we interrupted a floating point thread (PSR[EF] == 1) */ diff --git a/cpukit/score/cpu/sparc/sparc-access.S b/cpukit/score/cpu/sparc/sparc-access.S index 9397cb815b..277fb7e652 100644 --- a/cpukit/score/cpu/sparc/sparc-access.S +++ b/cpukit/score/cpu/sparc/sparc-access.S @@ -80,7 +80,9 @@ SYM(_st32): SYM(_st_be64): SYM(_st64): - std %o1, [%o0] + mov %o2, %o3 + mov %o1, %o2 + std %o2, [%o0] retl nop diff --git a/cpukit/score/cpu/sparc/sparc-counter-asm.S b/cpukit/score/cpu/sparc/sparc-counter-asm.S index a1e18ae5b7..fb7783e096 100644 --- a/cpukit/score/cpu/sparc/sparc-counter-asm.S +++ b/cpukit/score/cpu/sparc/sparc-counter-asm.S @@ -31,8 +31,9 @@ SYM(_SPARC_Counter_read_default): sethi %hi(_SPARC_Counter + 12), %o1 ld [%o1 + %lo(_SPARC_Counter + 12)], %o0 add %o0, 1, %o0 + st %o0, [%o1 + %lo(_SPARC_Counter + 12)] jmp %o7 + 8 - st %o0, [%o1 + %lo(_SPARC_Counter + 12)] + nop PUBLIC(_SPARC_Counter_read_up) PUBLIC(_SPARC_Get_timecount_up) diff --git a/cpukit/score/cpu/sparc/syscall.S b/cpukit/score/cpu/sparc/syscall.S index 4f4ef32c53..737a501098 100644 --- a/cpukit/score/cpu/sparc/syscall.S +++ b/cpukit/score/cpu/sparc/syscall.S @@ -218,9 +218,10 @@ SYM(syscall_lazy_fp_switch): .Lfp_save_done: /* Restore the floating point context if necessary */ + st %g0, [%l4 + %lo(SPARC_THREAD_CONTROL_REGISTERS_FP_CONTEXT_OFFSET)] cmp %l6, 0 be .Lfp_restore_done - st %g0, [%l4 + %lo(SPARC_THREAD_CONTROL_REGISTERS_FP_CONTEXT_OFFSET)] + nop ldd [%l6 + SPARC_FP_CONTEXT_OFFSET_F0_F1], %f0 ldd [%l6 + SPARC_FP_CONTEXT_OFFSET_F2_F3], %f2 ldd [%l6 + SPARC_FP_CONTEXT_OFFSET_F4_F5], %f4 diff --git a/cpukit/score/src/corebarrierrelease.c b/cpukit/score/src/corebarrierrelease.c index a2e68862b3..da8da0a7d5 100644 --- a/cpukit/score/src/corebarrierrelease.c +++ b/cpukit/score/src/corebarrierrelease.c @@ -27,10 +27,9 @@ uint32_t _CORE_barrier_Do_flush( Thread_queue_Context *queue_context ) { - the_barrier->number_of_waiting_threads = 0; return _Thread_queue_Flush_critical( &the_barrier->Wait_queue.Queue, - CORE_BARRIER_TQ_OPERATIONS, + &_CORE_barrier_Thread_queue_operations, filter, queue_context ); diff --git a/cpukit/score/src/corebarrierwait.c b/cpukit/score/src/corebarrierwait.c index f45873ccb8..2c59460c69 100644 --- a/cpukit/score/src/corebarrierwait.c +++ b/cpukit/score/src/corebarrierwait.c @@ -21,6 +21,36 @@ #include <rtems/score/corebarrierimpl.h> #include <rtems/score/statesimpl.h> #include <rtems/score/threadimpl.h> +#include <rtems/score/threadqops.h> + +static void _CORE_barrier_Thread_queue_extract( + Thread_queue_Queue *queue, + Thread_Control *the_thread, + Thread_queue_Context *queue_context +) +{ + CORE_barrier_Control *the_barrier; + + the_barrier = RTEMS_CONTAINER_OF( + queue, + CORE_barrier_Control, + Wait_queue.Queue + ); + --the_barrier->number_of_waiting_threads; + _Thread_queue_FIFO_extract( + &the_barrier->Wait_queue.Queue, + the_thread, + queue_context + ); +} + +const Thread_queue_Operations _CORE_barrier_Thread_queue_operations = { + .priority_actions = _Thread_queue_Do_nothing_priority_actions, + .enqueue = _Thread_queue_FIFO_enqueue, + .extract = _CORE_barrier_Thread_queue_extract, + .surrender = _Thread_queue_FIFO_surrender, + .first = _Thread_queue_FIFO_first +}; Status_Control _CORE_barrier_Seize( CORE_barrier_Control *the_barrier, @@ -50,7 +80,7 @@ Status_Control _CORE_barrier_Seize( ); _Thread_queue_Enqueue( &the_barrier->Wait_queue.Queue, - CORE_BARRIER_TQ_OPERATIONS, + &_CORE_barrier_Thread_queue_operations, executing, queue_context ); diff --git a/cpukit/score/src/heapallocate.c b/cpukit/score/src/heapallocate.c index 4b8b3f0de6..1c71e4a037 100644 --- a/cpukit/score/src/heapallocate.c +++ b/cpukit/score/src/heapallocate.c @@ -73,6 +73,15 @@ return search_again; } + + void _Heap_Protection_free_all_delayed_blocks( Heap_Control *heap ) + { + bool search_again; + + do { + search_again = _Heap_Protection_free_delayed_blocks( heap, 0 ); + } while ( search_again ); + } #endif #ifdef RTEMS_HEAP_DEBUG diff --git a/cpukit/score/src/kern_tc.c b/cpukit/score/src/kern_tc.c index 1b65cf41ee..56ec4751ce 100644 --- a/cpukit/score/src/kern_tc.c +++ b/cpukit/score/src/kern_tc.c @@ -1,4 +1,6 @@ /*- + * SPDX-License-Identifier: Beerware + * * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you @@ -7,7 +9,6 @@ * ---------------------------------------------------------------------------- * * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation - * All rights reserved. * * Portions of this software were developed by Julien Ridoux at the University * of Melbourne under sponsorship from the FreeBSD Foundation. @@ -44,9 +45,8 @@ #include <rtems/score/watchdogimpl.h> #endif /* __rtems__ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: head/sys/kern/kern_tc.c 324528 2017-10-11 11:03:11Z kib $"); +__FBSDID("$FreeBSD$"); -#include "opt_compat.h" #include "opt_ntp.h" #include "opt_ffclock.h" @@ -79,16 +79,7 @@ ISR_LOCK_DEFINE(, _Timecounter_Lock, "Timecounter") _ISR_lock_Release_and_ISR_enable(&_Timecounter_Lock, lock_context) #define hz rtems_clock_get_ticks_per_second() #define printf(...) -#define bcopy(x, y, z) memcpy(y, x, z); #define log(...) -static inline int -builtin_fls(int x) -{ - return x ? sizeof(x) * 8 - __builtin_clz(x) : 0; -} -#define fls(x) builtin_fls(x) -/* FIXME: https://devel.rtems.org/ticket/2348 */ -#define ntp_update_second(a, b) do { (void) a; (void) b; } while (0) static inline void atomic_thread_fence_acq(void) @@ -117,6 +108,24 @@ atomic_store_rel_int(Atomic_Uint *i, u_int val) _Atomic_Store_uint(i, val, ATOMIC_ORDER_RELEASE); } + +static inline void * +atomic_load_ptr(void *ptr) +{ + + return ((void *)_Atomic_Load_uintptr(ptr, ATOMIC_ORDER_RELAXED)); +} + +static Timecounter_NTP_update_second _Timecounter_NTP_update_second; + +void +_Timecounter_Set_NTP_update_second(Timecounter_NTP_update_second handler) +{ + + _Timecounter_NTP_update_second = handler; +} + +#define ntp_update_second(a, b) (*ntp_update_second_handler)(a, b) #endif /* __rtems__ */ /* @@ -158,6 +167,7 @@ struct timehands { struct timecounter *th_counter; int64_t th_adjustment; uint64_t th_scale; + uint32_t th_large_delta; uint32_t th_offset_count; struct bintime th_offset; struct bintime th_bintime; @@ -173,6 +183,40 @@ struct timehands { struct timehands *th_next; }; +#ifndef __rtems__ +static struct timehands ths[16] = { + [0] = { + .th_counter = &dummy_timecounter, + .th_scale = (uint64_t)-1 / 1000000, + .th_large_delta = 1000000, + .th_offset = { .sec = 1 }, + .th_generation = 1, + }, +}; + +static struct timehands *volatile timehands = &ths[0]; +struct timecounter *timecounter = &dummy_timecounter; +static struct timecounter *timecounters = &dummy_timecounter; + +/* Mutex to protect the timecounter list. */ +static struct mtx tc_lock; +MTX_SYSINIT(tc_lock, &tc_lock, "tc", MTX_DEF); + +int tc_min_ticktock_freq = 1; +#else /* __rtems__ */ +/* + * In FreeBSD, the timehands count is a tuning option from two to 16. The + * tuning option was added since it is inexpensive and some FreeBSD users asked + * for it to play around. The default value is two. One system which did not + * work with two timehands was a system with one processor and a specific PPS + * device. + * + * For RTEMS, in uniprocessor configurations, just use one timehand since the + * update is done with interrupt disabled. + * + * In SMP configurations, use a fixed set of two timehands until someone + * reports an issue. + */ #if defined(RTEMS_SMP) static struct timehands th0; static struct timehands th1 = { @@ -183,7 +227,8 @@ static struct timehands th0 = { .th_counter = &dummy_timecounter, .th_scale = (uint64_t)-1 / 1000000, .th_offset = { .sec = 1 }, - .th_generation = 1, + .th_large_delta = 1000000, + .th_generation = UINT_MAX, #ifdef __rtems__ .th_bintime = { .sec = TOD_SECONDS_1970_THROUGH_1988 }, .th_microtime = { TOD_SECONDS_1970_THROUGH_1988, 0 }, @@ -199,10 +244,6 @@ static struct timehands th0 = { static struct timehands *volatile timehands = &th0; struct timecounter *timecounter = &dummy_timecounter; -#ifndef __rtems__ -static struct timecounter *timecounters = &dummy_timecounter; - -int tc_min_ticktock_freq = 1; #endif /* __rtems__ */ #ifndef __rtems__ @@ -214,17 +255,33 @@ volatile int32_t time_uptime = 1; #endif /* __rtems__ */ #ifndef __rtems__ +/* + * The system time is always computed by summing the estimated boot time and the + * system uptime. The timehands track boot time, but it changes when the system + * time is set by the user, stepped by ntpd or adjusted when resuming. It + * is set to new_time - uptime. + */ static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); -SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD, - NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime"); +SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, + CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, + sysctl_kern_boottime, "S,timeval", + "Estimated system boottime"); -SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); -static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, ""); +SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, + ""); +static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, + CTLFLAG_RW | CTLFLAG_MPSAFE, 0, + ""); static int timestepwarnings; -SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, +SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RWTUN, ×tepwarnings, 0, "Log time steps"); +static int timehands_count = 2; +SYSCTL_INT(_kern_timecounter, OID_AUTO, timehands_count, + CTLFLAG_RDTUN | CTLFLAG_NOFETCH, + &timehands_count, 0, "Count of timehands in rotation"); + struct bintime bt_timethreshold; struct bintime bt_tickthreshold; sbintime_t sbt_timethreshold; @@ -242,6 +299,7 @@ SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation, volatile int rtc_generation = 1; static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */ +static char tc_from_tunable[16]; #endif /* __rtems__ */ static void tc_windup(struct bintime *new_boottimebin); @@ -253,6 +311,7 @@ static void _Timecounter_Windup(struct bintime *new_boottimebin, #endif /* __rtems__ */ void dtrace_getnanotime(struct timespec *tsp); +void dtrace_getnanouptime(struct timespec *tsp); #ifndef __rtems__ static int @@ -262,7 +321,8 @@ sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) getboottime(&boottime); -#ifndef __mips__ +/* i386 is the only arch which uses a 32bits time_t */ +#ifdef __amd64__ #ifdef SCTL_MASK32 int tv[2]; @@ -317,20 +377,85 @@ tc_delta(struct timehands *th) * the comment in <sys/time.h> for a description of these 12 functions. */ -#ifdef FFCLOCK -void -fbclock_binuptime(struct bintime *bt) +static __inline void +bintime_off(struct bintime *bt, u_int off) { struct timehands *th; - unsigned int gen; + struct bintime *btp; + uint64_t scale, x; +#ifndef __rtems__ + u_int delta, gen, large_delta; +#else /* __rtems__ */ + uint32_t delta, large_delta; + u_int gen; +#endif /* __rtems__ */ do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_offset; - bintime_addx(bt, th->th_scale * tc_delta(th)); + btp = (struct bintime *)((vm_offset_t)th + off); + *bt = *btp; + scale = th->th_scale; + delta = tc_delta(th); + large_delta = th->th_large_delta; atomic_thread_fence_acq(); +#if defined(RTEMS_SMP) } while (gen == 0 || gen != th->th_generation); +#else + } while (gen != th->th_generation); +#endif + + if (__predict_false(delta >= large_delta)) { + /* Avoid overflow for scale * delta. */ + x = (scale >> 32) * delta; + bt->sec += x >> 32; + bintime_addx(bt, x << 32); + bintime_addx(bt, (scale & 0xffffffff) * delta); + } else { + bintime_addx(bt, scale * delta); + } +} +#define GETTHBINTIME(dst, member) \ +do { \ + _Static_assert(_Generic(((struct timehands *)NULL)->member, \ + struct bintime: 1, default: 0) == 1, \ + "struct timehands member is not of struct bintime type"); \ + bintime_off(dst, __offsetof(struct timehands, member)); \ +} while (0) + +static __inline void +getthmember(void *out, size_t out_size, u_int off) +{ + struct timehands *th; + u_int gen; + + do { + th = timehands; + gen = atomic_load_acq_int(&th->th_generation); + memcpy(out, (char *)th + off, out_size); + atomic_thread_fence_acq(); +#if defined(RTEMS_SMP) + } while (gen == 0 || gen != th->th_generation); +#else + } while (gen != th->th_generation); +#endif +} +#define GETTHMEMBER(dst, member) \ +do { \ + _Static_assert(_Generic(*dst, \ + __typeof(((struct timehands *)NULL)->member): 1, \ + default: 0) == 1, \ + "*dst and struct timehands member have different types"); \ + getthmember(dst, sizeof(*dst), __offsetof(struct timehands, \ + member)); \ +} while (0) + +#ifdef FFCLOCK +void +fbclock_binuptime(struct bintime *bt) +{ + + GETTHBINTIME(bt, th_offset); } void @@ -354,16 +479,8 @@ fbclock_microuptime(struct timeval *tvp) void fbclock_bintime(struct bintime *bt) { - struct timehands *th; - unsigned int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_bintime; - bintime_addx(bt, th->th_scale * tc_delta(th)); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHBINTIME(bt, th_bintime); } void @@ -387,116 +504,88 @@ fbclock_microtime(struct timeval *tvp) void fbclock_getbinuptime(struct bintime *bt) { - struct timehands *th; - unsigned int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_offset; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(bt, th_offset); } void fbclock_getnanouptime(struct timespec *tsp) { - struct timehands *th; - unsigned int gen; + struct bintime bt; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - bintime2timespec(&th->th_offset, tsp); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(&bt, th_offset); + bintime2timespec(&bt, tsp); } void fbclock_getmicrouptime(struct timeval *tvp) { - struct timehands *th; - unsigned int gen; + struct bintime bt; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - bintime2timeval(&th->th_offset, tvp); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(&bt, th_offset); + bintime2timeval(&bt, tvp); } void fbclock_getbintime(struct bintime *bt) { - struct timehands *th; - unsigned int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_bintime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(bt, th_bintime); } void fbclock_getnanotime(struct timespec *tsp) { - struct timehands *th; - unsigned int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *tsp = th->th_nanotime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(tsp, th_nanotime); } void fbclock_getmicrotime(struct timeval *tvp) { - struct timehands *th; - unsigned int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *tvp = th->th_microtime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(tvp, th_microtime); } #else /* !FFCLOCK */ + void binuptime(struct bintime *bt) { - struct timehands *th; - uint32_t gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_offset; - bintime_addx(bt, th->th_scale * tc_delta(th)); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHBINTIME(bt, th_offset); } #ifdef __rtems__ sbintime_t _Timecounter_Sbinuptime(void) { struct timehands *th; - uint32_t gen; sbintime_t sbt; + uint64_t scale; + uint32_t delta; + uint32_t large_delta; + u_int gen; do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); sbt = bttosbt(th->th_offset); - sbt += (th->th_scale * tc_delta(th)) >> 32; + scale = th->th_scale; + delta = tc_delta(th); + large_delta = th->th_large_delta; atomic_thread_fence_acq(); +#if defined(RTEMS_SMP) } while (gen == 0 || gen != th->th_generation); +#else + } while (gen != th->th_generation); +#endif + + if (__predict_false(delta >= large_delta)) { + /* Avoid overflow for scale * delta. */ + sbt += (scale >> 32) * delta; + sbt += ((scale & 0xffffffff) * delta) >> 32; + } else { + sbt += (scale * delta) >> 32; + } return (sbt); } @@ -523,16 +612,8 @@ microuptime(struct timeval *tvp) void bintime(struct bintime *bt) { - struct timehands *th; - u_int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_bintime; - bintime_addx(bt, th->th_scale * tc_delta(th)); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHBINTIME(bt, th_bintime); } void @@ -556,85 +637,47 @@ microtime(struct timeval *tvp) void getbinuptime(struct bintime *bt) { - struct timehands *th; - uint32_t gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_offset; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(bt, th_offset); } void getnanouptime(struct timespec *tsp) { - struct timehands *th; - uint32_t gen; + struct bintime bt; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - bintime2timespec(&th->th_offset, tsp); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(&bt, th_offset); + bintime2timespec(&bt, tsp); } void getmicrouptime(struct timeval *tvp) { - struct timehands *th; - uint32_t gen; + struct bintime bt; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - bintime2timeval(&th->th_offset, tvp); - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(&bt, th_offset); + bintime2timeval(&bt, tvp); } void getbintime(struct bintime *bt) { - struct timehands *th; - uint32_t gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *bt = th->th_bintime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(bt, th_bintime); } void getnanotime(struct timespec *tsp) { - struct timehands *th; - uint32_t gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *tsp = th->th_nanotime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(tsp, th_nanotime); } void getmicrotime(struct timeval *tvp) { - struct timehands *th; - uint32_t gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *tvp = th->th_microtime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(tvp, th_microtime); } #endif /* FFCLOCK */ @@ -650,15 +693,8 @@ getboottime(struct timeval *boottime) void getboottimebin(struct bintime *boottimebin) { - struct timehands *th; - u_int gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *boottimebin = th->th_boottime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(boottimebin, th_boottime); } #ifdef FFCLOCK @@ -1175,15 +1211,22 @@ getmicrotime(struct timeval *tvp) void dtrace_getnanotime(struct timespec *tsp) { - struct timehands *th; - uint32_t gen; - do { - th = timehands; - gen = atomic_load_acq_int(&th->th_generation); - *tsp = th->th_nanotime; - atomic_thread_fence_acq(); - } while (gen == 0 || gen != th->th_generation); + GETTHMEMBER(tsp, th_nanotime); +} + +/* + * This is a clone of getnanouptime used for time since boot. + * The dtrace_ prefix prevents fbt from creating probes for + * it so an uptime that can be safely used in all fbt probes. + */ +void +dtrace_getnanouptime(struct timespec *tsp) +{ + struct bintime bt; + + GETTHMEMBER(&bt, th_offset); + bintime2timespec(&bt, tsp); } #endif /* __rtems__ */ @@ -1364,26 +1407,32 @@ tc_init(struct timecounter *tc) tc->tc_quality); } - tc->tc_next = timecounters; - timecounters = tc; /* * Set up sysctl tree for this counter. */ tc_root = SYSCTL_ADD_NODE_WITH_LABEL(NULL, SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, - CTLFLAG_RW, 0, "timecounter description", "timecounter"); + CTLFLAG_RW | CTLFLAG_MPSAFE, 0, + "timecounter description", "timecounter"); SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, "mask for implemented bits"); SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, - "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc), - sysctl_kern_timecounter_get, "IU", "current timecounter value"); + "counter", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, tc, + sizeof(*tc), sysctl_kern_timecounter_get, "IU", + "current timecounter value"); SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, - "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc), - sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); + "frequency", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, tc, + sizeof(*tc), sysctl_kern_timecounter_freq, "QU", + "timecounter frequency"); SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, "quality", CTLFLAG_RD, &(tc->tc_quality), 0, "goodness of time counter"); + + mtx_lock(&tc_lock); + tc->tc_next = timecounters; + timecounters = tc; + /* * Do not automatically switch if the current tc was specifically * chosen. Never automatically use a timecounter with negative quality. @@ -1391,21 +1440,31 @@ tc_init(struct timecounter *tc) * worse since this timecounter may not be monotonic. */ if (tc_chosen) - return; + goto unlock; if (tc->tc_quality < 0) - return; -#endif /* __rtems__ */ + goto unlock; + if (tc_from_tunable[0] != '\0' && + strcmp(tc->tc_name, tc_from_tunable) == 0) { + tc_chosen = 1; + tc_from_tunable[0] = '\0'; + } else { + if (tc->tc_quality < timecounter->tc_quality) + goto unlock; + if (tc->tc_quality == timecounter->tc_quality && + tc->tc_frequency < timecounter->tc_frequency) + goto unlock; + } + (void)tc->tc_get_timecount(tc); + timecounter = tc; +unlock: + mtx_unlock(&tc_lock); +#else /* __rtems__ */ if (tc->tc_quality < timecounter->tc_quality) return; if (tc->tc_quality == timecounter->tc_quality && tc->tc_frequency < timecounter->tc_frequency) return; -#ifndef __rtems__ - (void)tc->tc_get_timecount(tc); - (void)tc->tc_get_timecount(tc); -#endif /* __rtems__ */ timecounter = tc; -#ifdef __rtems__ tc_windup(NULL); #endif /* __rtems__ */ } @@ -1493,6 +1552,40 @@ _Timecounter_Set_clock(const struct bintime *_bt, } /* + * Recalculate the scaling factor. We want the number of 1/2^64 + * fractions of a second per period of the hardware counter, taking + * into account the th_adjustment factor which the NTP PLL/adjtime(2) + * processing provides us with. + * + * The th_adjustment is nanoseconds per second with 32 bit binary + * fraction and we want 64 bit binary fraction of second: + * + * x = a * 2^32 / 10^9 = a * 4.294967296 + * + * The range of th_adjustment is +/- 5000PPM so inside a 64bit int + * we can only multiply by about 850 without overflowing, that + * leaves no suitably precise fractions for multiply before divide. + * + * Divide before multiply with a fraction of 2199/512 results in a + * systematic undercompensation of 10PPM of th_adjustment. On a + * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. + * + * We happily sacrifice the lowest of the 64 bits of our result + * to the goddess of code clarity. + */ +static void +recalculate_scaling_factor_and_large_delta(struct timehands *th) +{ + uint64_t scale; + + scale = (uint64_t)1 << 63; + scale += (th->th_adjustment / 1024) * 2199; + scale /= th->th_counter->tc_frequency; + th->th_scale = scale * 2; + th->th_large_delta = MIN(((uint64_t)1 << 63) / scale, UINT_MAX); +} + +/* * Initialize the next struct timehands in the ring and make * it the active timehands. Along the way we might switch to a different * timecounter and/or do seconds processing in NTP. Slightly magic. @@ -1513,11 +1606,17 @@ _Timecounter_Windup(struct bintime *new_boottimebin, #endif /* __rtems__ */ { struct bintime bt; + struct timecounter *tc; struct timehands *th, *tho; - uint64_t scale; - uint32_t delta, ncount, ogen; + uint32_t delta, ncount; +#if defined(RTEMS_SMP) + u_int ogen; +#endif int i; time_t t; +#ifdef __rtems__ + Timecounter_NTP_update_second ntp_update_second_handler; +#endif /* * Make the next timehands a copy of the current one, but do @@ -1531,14 +1630,12 @@ _Timecounter_Windup(struct bintime *new_boottimebin, tho = timehands; #if defined(RTEMS_SMP) th = tho->th_next; -#else - th = tho; -#endif ogen = th->th_generation; th->th_generation = 0; atomic_thread_fence_rel(); -#if defined(RTEMS_SMP) - bcopy(tho, th, offsetof(struct timehands, th_generation)); + memcpy(th, tho, offsetof(struct timehands, th_generation)); +#else + th = tho; #endif if (new_boottimebin != NULL) th->th_boottime = *new_boottimebin; @@ -1548,9 +1645,10 @@ _Timecounter_Windup(struct bintime *new_boottimebin, * changing timecounters, a counter value from the new timecounter. * Update the offset fields accordingly. */ + tc = atomic_load_ptr(&timecounter); delta = tc_delta(th); - if (th->th_counter != timecounter) - ncount = timecounter->tc_get_timecount(timecounter); + if (th->th_counter != tc) + ncount = tc->tc_get_timecount(tc); else ncount = 0; #ifdef FFCLOCK @@ -1584,7 +1682,7 @@ _Timecounter_Windup(struct bintime *new_boottimebin, #endif /* __rtems__ */ /* - * Deal with NTP second processing. The for loop normally + * Deal with NTP second processing. The loop normally * iterates at most once, but in extreme situations it might * keep NTP sane if timeouts are not run for several seconds. * At boot, the time step can be large when the TOD hardware @@ -1594,69 +1692,57 @@ _Timecounter_Windup(struct bintime *new_boottimebin, */ bt = th->th_offset; bintime_add(&bt, &th->th_boottime); +#ifdef __rtems__ + ntp_update_second_handler = _Timecounter_NTP_update_second; + if (ntp_update_second_handler != NULL) { +#endif /* __rtems__ */ i = bt.sec - tho->th_microtime.tv_sec; - if (i > LARGE_STEP) - i = 2; - for (; i > 0; i--) { - t = bt.sec; - ntp_update_second(&th->th_adjustment, &bt.sec); - if (bt.sec != t) - th->th_boottime.sec += bt.sec - t; + if (i > 0) { + if (i > LARGE_STEP) + i = 2; + + do { + t = bt.sec; + ntp_update_second(&th->th_adjustment, &bt.sec); + if (bt.sec != t) + th->th_boottime.sec += bt.sec - t; + --i; + } while (i > 0); + + recalculate_scaling_factor_and_large_delta(th); } +#ifdef __rtems__ + } +#endif /* __rtems__ */ + /* Update the UTC timestamps used by the get*() functions. */ th->th_bintime = bt; bintime2timeval(&bt, &th->th_microtime); bintime2timespec(&bt, &th->th_nanotime); /* Now is a good time to change timecounters. */ - if (th->th_counter != timecounter) { + if (th->th_counter != tc) { #ifndef __rtems__ #ifndef __arm__ - if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0) + if ((tc->tc_flags & TC_FLAGS_C2STOP) != 0) cpu_disable_c2_sleep++; if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0) cpu_disable_c2_sleep--; #endif #endif /* __rtems__ */ - th->th_counter = timecounter; + th->th_counter = tc; th->th_offset_count = ncount; #ifndef __rtems__ - tc_min_ticktock_freq = max(1, timecounter->tc_frequency / - (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); + tc_min_ticktock_freq = max(1, tc->tc_frequency / + (((uint64_t)tc->tc_counter_mask + 1) / 3)); #endif /* __rtems__ */ + recalculate_scaling_factor_and_large_delta(th); #ifdef FFCLOCK ffclock_change_tc(th); #endif } - /*- - * Recalculate the scaling factor. We want the number of 1/2^64 - * fractions of a second per period of the hardware counter, taking - * into account the th_adjustment factor which the NTP PLL/adjtime(2) - * processing provides us with. - * - * The th_adjustment is nanoseconds per second with 32 bit binary - * fraction and we want 64 bit binary fraction of second: - * - * x = a * 2^32 / 10^9 = a * 4.294967296 - * - * The range of th_adjustment is +/- 5000PPM so inside a 64bit int - * we can only multiply by about 850 without overflowing, that - * leaves no suitably precise fractions for multiply before divide. - * - * Divide before multiply with a fraction of 2199/512 results in a - * systematic undercompensation of 10PPM of th_adjustment. On a - * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. - * - * We happily sacrifice the lowest of the 64 bits of our result - * to the goddess of code clarity. - * - */ - scale = (uint64_t)1 << 63; - scale += (th->th_adjustment / 1024) * 2199; - scale /= th->th_counter->tc_frequency; - th->th_scale = scale * 2; - +#if defined(RTEMS_SMP) /* * Now that the struct timehands is again consistent, set the new * generation number, making sure to not make it zero. @@ -1664,6 +1750,9 @@ _Timecounter_Windup(struct bintime *new_boottimebin, if (++ogen == 0) ogen = 1; atomic_store_rel_int(&th->th_generation, ogen); +#else + atomic_store_rel_int(&th->th_generation, th->th_generation + 1); +#endif /* Go live with the new struct timehands. */ #ifdef FFCLOCK @@ -1701,23 +1790,28 @@ sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) struct timecounter *newtc, *tc; int error; + mtx_lock(&tc_lock); tc = timecounter; strlcpy(newname, tc->tc_name, sizeof(newname)); + mtx_unlock(&tc_lock); error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); if (error != 0 || req->newptr == NULL) return (error); + + mtx_lock(&tc_lock); /* Record that the tc in use now was specifically chosen. */ tc_chosen = 1; - if (strcmp(newname, tc->tc_name) == 0) + if (strcmp(newname, tc->tc_name) == 0) { + mtx_unlock(&tc_lock); return (0); + } for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { if (strcmp(newname, newtc->tc_name) != 0) continue; /* Warm up new timecounter. */ (void)newtc->tc_get_timecount(newtc); - (void)newtc->tc_get_timecount(newtc); timecounter = newtc; @@ -1729,16 +1823,16 @@ sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) * use any locking and that it can be called in hard interrupt * context via 'tc_windup()'. */ - return (0); + break; } - return (EINVAL); + mtx_unlock(&tc_lock); + return (newtc != NULL ? 0 : EINVAL); } - -SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, - 0, 0, sysctl_kern_timecounter_hardware, "A", +SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, + CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, 0, 0, + sysctl_kern_timecounter_hardware, "A", "Timecounter hardware selected"); - /* Report the available timecounter hardware. */ static int sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) @@ -1747,19 +1841,26 @@ sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) struct timecounter *tc; int error; + error = sysctl_wire_old_buffer(req, 0); + if (error != 0) + return (error); sbuf_new_for_sysctl(&sb, NULL, 0, req); + mtx_lock(&tc_lock); for (tc = timecounters; tc != NULL; tc = tc->tc_next) { if (tc != timecounters) sbuf_putc(&sb, ' '); sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality); } + mtx_unlock(&tc_lock); error = sbuf_finish(&sb); sbuf_delete(&sb); return (error); } -SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, - 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); +SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0, + sysctl_kern_timecounter_choice, "A", + "Timecounter hardware detected"); #endif /* __rtems__ */ #ifndef __rtems__ @@ -1803,10 +1904,10 @@ pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps) tv.tv_usec = fapi->timeout.tv_nsec / 1000; timo = tvtohz(&tv); } - aseq = pps->ppsinfo.assert_sequence; - cseq = pps->ppsinfo.clear_sequence; - while (aseq == pps->ppsinfo.assert_sequence && - cseq == pps->ppsinfo.clear_sequence) { + aseq = atomic_load_int(&pps->ppsinfo.assert_sequence); + cseq = atomic_load_int(&pps->ppsinfo.clear_sequence); + while (aseq == atomic_load_int(&pps->ppsinfo.assert_sequence) && + cseq == atomic_load_int(&pps->ppsinfo.clear_sequence)) { if (abi_aware(pps, 1) && pps->driver_mtx != NULL) { if (pps->flags & PPSFLAG_MTX_SPIN) { err = msleep_spin(pps, pps->driver_mtx, @@ -2144,27 +2245,38 @@ _Timecounter_Tick_simple(uint32_t delta, uint32_t offset, { struct bintime bt; struct timehands *th; - uint32_t ogen; +#if defined(RTEMS_SMP) + u_int ogen; +#endif th = timehands; +#if defined(RTEMS_SMP) ogen = th->th_generation; + th->th_generation = 0; + atomic_thread_fence_rel(); +#endif + th->th_offset_count = offset; bintime_addx(&th->th_offset, th->th_scale * delta); - bt = th->th_offset; bintime_add(&bt, &th->th_boottime); + /* Update the UTC timestamps used by the get*() functions. */ th->th_bintime = bt; bintime2timeval(&bt, &th->th_microtime); bintime2timespec(&bt, &th->th_nanotime); +#if defined(RTEMS_SMP) /* * Now that the struct timehands is again consistent, set the new * generation number, making sure to not make it zero. */ if (++ogen == 0) ogen = 1; - th->th_generation = ogen; + atomic_store_rel_int(&th->th_generation, ogen); +#else + atomic_store_rel_int(&th->th_generation, th->th_generation + 1); +#endif /* Go live with the new struct timehands. */ time_second = th->th_microtime.tv_sec; @@ -2218,6 +2330,28 @@ done: return (0); } +/* Set up the requested number of timehands. */ +static void +inittimehands(void *dummy) +{ + struct timehands *thp; + int i; + + TUNABLE_INT_FETCH("kern.timecounter.timehands_count", + &timehands_count); + if (timehands_count < 1) + timehands_count = 1; + if (timehands_count > nitems(ths)) + timehands_count = nitems(ths); + for (i = 1, thp = &ths[0]; i < timehands_count; thp = &ths[i++]) + thp->th_next = &ths[i]; + thp->th_next = &ths[0]; + + TUNABLE_STR_FETCH("kern.timecounter.hardware", tc_from_tunable, + sizeof(tc_from_tunable)); +} +SYSINIT(timehands, SI_SUB_TUNABLES, SI_ORDER_ANY, inittimehands, NULL); + static void inittimecounter(void *dummy) { @@ -2248,9 +2382,9 @@ inittimecounter(void *dummy) #ifdef FFCLOCK ffclock_init(); #endif + /* warm up new timecounter (again) and get rolling. */ (void)timecounter->tc_get_timecount(timecounter); - (void)timecounter->tc_get_timecount(timecounter); mtx_lock_spin(&tc_setclock_mtx); tc_windup(NULL); mtx_unlock_spin(&tc_setclock_mtx); @@ -2263,8 +2397,8 @@ SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); static int cpu_tick_variable; static uint64_t cpu_tick_frequency; -static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base); -static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last); +DPCPU_DEFINE_STATIC(uint64_t, tc_cpu_ticks_base); +DPCPU_DEFINE_STATIC(unsigned, tc_cpu_ticks_last); static uint64_t tc_cpu_ticks(void) @@ -2438,7 +2572,6 @@ tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) enabled = 0; return (enabled); } -#endif /* __rtems__ */ #ifdef COMPAT_FREEBSD32 uint32_t @@ -2465,3 +2598,34 @@ tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) return (enabled); } #endif + +#include "opt_ddb.h" +#ifdef DDB +#include <ddb/ddb.h> + +DB_SHOW_COMMAND(timecounter, db_show_timecounter) +{ + struct timehands *th; + struct timecounter *tc; + u_int val1, val2; + + th = timehands; + tc = th->th_counter; + val1 = tc->tc_get_timecount(tc); + __compiler_membar(); + val2 = tc->tc_get_timecount(tc); + + db_printf("timecounter %p %s\n", tc, tc->tc_name); + db_printf(" mask %#x freq %ju qual %d flags %#x priv %p\n", + tc->tc_counter_mask, (uintmax_t)tc->tc_frequency, tc->tc_quality, + tc->tc_flags, tc->tc_priv); + db_printf(" val %#x %#x\n", val1, val2); + db_printf("timehands adj %#jx scale %#jx ldelta %d off_cnt %d gen %d\n", + (uintmax_t)th->th_adjustment, (uintmax_t)th->th_scale, + th->th_large_delta, th->th_offset_count, th->th_generation); + db_printf(" offset %jd %jd boottime %jd %jd\n", + (intmax_t)th->th_offset.sec, (uintmax_t)th->th_offset.frac, + (intmax_t)th->th_boottime.sec, (uintmax_t)th->th_boottime.frac); +} +#endif +#endif /* __rtems__ */ diff --git a/cpukit/score/src/objectactivecount.c b/cpukit/score/src/objectactivecount.c index c658fc21e3..12c15147c7 100644 --- a/cpukit/score/src/objectactivecount.c +++ b/cpukit/score/src/objectactivecount.c @@ -24,14 +24,22 @@ Objects_Maximum _Objects_Active_count( const Objects_Information *information ) { - Objects_Maximum inactive; - Objects_Maximum maximum; + Objects_Maximum active; + Objects_Maximum index; + Objects_Maximum maximum; + Objects_Control **local_table; _Assert( _Objects_Allocator_is_owner() ); - inactive = (Objects_Maximum) - _Chain_Node_count_unprotected( &information->Inactive ); + active = 0; maximum = _Objects_Get_maximum_index( information ); + local_table = information->local_table; - return maximum - inactive; + for ( index = 0; index < maximum; ++index ) { + if ( local_table[ index ] != NULL ) { + ++active; + } + } + + return active; } diff --git a/cpukit/score/src/objectfree.c b/cpukit/score/src/objectfree.c index 1f7aa37e3f..964adbc62a 100644 --- a/cpukit/score/src/objectfree.c +++ b/cpukit/score/src/objectfree.c @@ -30,14 +30,16 @@ void _Objects_Free_unlimited( if ( _Objects_Is_auto_extend( information ) ) { Objects_Maximum objects_per_block; - Objects_Maximum block; - Objects_Maximum inactive; + Objects_Maximum index; objects_per_block = information->objects_per_block; - block = _Objects_Get_index( the_object->id ) - OBJECTS_INDEX_MINIMUM; + index = _Objects_Get_index( the_object->id ) - OBJECTS_INDEX_MINIMUM; - if ( block > objects_per_block ) { - block /= objects_per_block; + if ( _Objects_Is_in_allocated_block( index, objects_per_block ) ) { + Objects_Maximum block; + Objects_Maximum inactive; + + block = index / objects_per_block; ++information->inactive_per_block[ block ]; diff --git a/cpukit/score/src/stackallocatorforidle.c b/cpukit/score/src/stackallocatorforidle.c new file mode 100644 index 0000000000..7c4fd10c7d --- /dev/null +++ b/cpukit/score/src/stackallocatorforidle.c @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (C) 2021 OAR Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <rtems/score/stack.h> +#include <rtems/score/thread.h> + +/** + * @brief Default stack allocator allocate for idle handler. + * + * The allocate for idle handler is optional even when the user thread stack + * allocator and deallocator are configured. + * + * The default allocator for IDLE thread stacks gets the memory from a + * statically allocated area provided via confdefs.h. + * + * @param cpu Index of the CPU for the IDLE thread using this stack + * @param stack_size The size of the stack area to allocate in bytes. + * + * @retval NULL Not enough memory (never returned). + * @retval other Pointer to begin of stack area. + */ +static void *_Stack_Allocator_allocate_for_idle_default( + uint32_t cpu, + size_t stack_size +) +{ + return &_Thread_Idle_stacks[ cpu * stack_size ]; +} + +const Stack_Allocator_allocate_for_idle _Stack_Allocator_allocate_for_idle = + _Stack_Allocator_allocate_for_idle_default; diff --git a/cpukit/score/src/threadcreateidle.c b/cpukit/score/src/threadcreateidle.c index 1e18ad07cc..e888aa111f 100644 --- a/cpukit/score/src/threadcreateidle.c +++ b/cpukit/score/src/threadcreateidle.c @@ -53,9 +53,15 @@ static void _Thread_Create_idle_for_CPU( Per_CPU_Control *cpu ) config.is_preemptible = true; config.stack_size = _Thread_Idle_stack_size + CPU_IDLE_TASK_IS_FP * CONTEXT_FP_SIZE; - config.stack_area = &_Thread_Idle_stacks[ - _Per_CPU_Get_index( cpu ) * config.stack_size - ]; + + /* + * The IDLE thread stacks may be statically allocated or there may be a + * custom allocator provided just as with user threads. + */ + config.stack_area = (*_Stack_Allocator_allocate_for_idle)( + _Per_CPU_Get_index( cpu ), + config.stack_size + ); /* * The entire workspace is zeroed during its initialization. Thus, all diff --git a/cpukit/score/src/threadqops.c b/cpukit/score/src/threadqops.c index 9894c3628b..8c3e0cb1dc 100644 --- a/cpukit/score/src/threadqops.c +++ b/cpukit/score/src/threadqops.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2016 embedded brains GmbH. All rights reserved. + * Copyright (c) 2015, 2021 embedded brains GmbH. All rights reserved. * * embedded brains GmbH * Dornierstr. 4 @@ -16,6 +16,7 @@ #include "config.h" #endif +#include <rtems/score/threadqops.h> #include <rtems/score/threadimpl.h> #include <rtems/score/assert.h> #include <rtems/score/chainimpl.h> @@ -38,7 +39,7 @@ Queue \ ) -static void _Thread_queue_Do_nothing_priority_actions( +void _Thread_queue_Do_nothing_priority_actions( Thread_queue_Queue *queue, Priority_Actions *priority_actions ) @@ -187,7 +188,7 @@ static void _Thread_queue_FIFO_do_extract( _Chain_Extract_unprotected( &scheduler_node->Wait.Priority.Node.Node.Chain ); } -static void _Thread_queue_FIFO_enqueue( +void _Thread_queue_FIFO_enqueue( Thread_queue_Queue *queue, Thread_Control *the_thread, Thread_queue_Context *queue_context @@ -202,7 +203,7 @@ static void _Thread_queue_FIFO_enqueue( ); } -static void _Thread_queue_FIFO_extract( +void _Thread_queue_FIFO_extract( Thread_queue_Queue *queue, Thread_Control *the_thread, Thread_queue_Context *queue_context @@ -218,23 +219,21 @@ static void _Thread_queue_FIFO_extract( ); } -static Thread_Control *_Thread_queue_FIFO_first( - Thread_queue_Heads *heads -) +Thread_Control *_Thread_queue_FIFO_first( const Thread_queue_Heads *heads ) { - Chain_Control *fifo; - Chain_Node *first; - Scheduler_Node *scheduler_node; + const Chain_Control *fifo; + const Chain_Node *first; + const Scheduler_Node *scheduler_node; fifo = &heads->Heads.Fifo; _Assert( !_Chain_Is_empty( fifo ) ); - first = _Chain_First( fifo ); + first = _Chain_Immutable_first( fifo ); scheduler_node = SCHEDULER_NODE_OF_WAIT_PRIORITY_NODE( first ); return _Scheduler_Node_get_owner( scheduler_node ); } -static Thread_Control *_Thread_queue_FIFO_surrender( +Thread_Control *_Thread_queue_FIFO_surrender( Thread_queue_Queue *queue, Thread_queue_Heads *heads, Thread_Control *previous_owner, @@ -243,6 +242,8 @@ static Thread_Control *_Thread_queue_FIFO_surrender( { Thread_Control *first; + (void) previous_owner; + first = _Thread_queue_FIFO_first( heads ); _Thread_queue_Queue_extract( queue, @@ -589,7 +590,7 @@ static void _Thread_queue_Priority_extract( } static Thread_Control *_Thread_queue_Priority_first( - Thread_queue_Heads *heads + const Thread_queue_Heads *heads ) { Thread_queue_Priority_queue *priority_queue; |