RTEMS: cpukit/include/rtems/score/percpu.h Source File

| RTEMS 5.4-rc2

Loading...
Searching...
No Matches
percpu.h
Go to the documentation of this file.
1
8/*
9 * COPYRIGHT (c) 1989-2011.
10 * On-Line Applications Research Corporation (OAR).
11 *
12 * Copyright (c) 2012, 2018 embedded brains GmbH
13 *
14 * The license and distribution terms for this file may be
15 * found in the file LICENSE in this distribution or at
16 * http://www.rtems.org/license/LICENSE.
17 */
18
19#ifndef _RTEMS_PERCPU_H
20#define _RTEMS_PERCPU_H
21
22#include <rtems/score/cpuimpl.h>
23
24#if defined( ASM )
25 #include <rtems/asm.h>
26#else
27 #include <rtems/score/assert.h>
28 #include <rtems/score/chain.h>
29 #include <rtems/score/isrlock.h>
30 #include <rtems/score/smp.h>
31 #include <rtems/score/timestamp.h>
32 #include <rtems/score/watchdog.h>
33#endif
34
35#ifdef __cplusplus
36extern "C" {
37#endif
38
39#if defined(RTEMS_SMP)
40 #if defined(RTEMS_PROFILING)
41 #define PER_CPU_CONTROL_SIZE_APPROX \
42 ( 512 + CPU_PER_CPU_CONTROL_SIZE + CPU_INTERRUPT_FRAME_SIZE )
43 #elif defined(RTEMS_DEBUG) || CPU_SIZEOF_POINTER > 4
44 #define PER_CPU_CONTROL_SIZE_APPROX \
45 ( 256 + CPU_PER_CPU_CONTROL_SIZE + CPU_INTERRUPT_FRAME_SIZE )
46 #else
47 #define PER_CPU_CONTROL_SIZE_APPROX \
48 ( 180 + CPU_PER_CPU_CONTROL_SIZE + CPU_INTERRUPT_FRAME_SIZE )
49 #endif
50
51 /*
52 * This ensures that on SMP configurations the individual per-CPU controls
53 * are on different cache lines to prevent false sharing. This define can be
54 * used in assembler code to easily get the per-CPU control for a particular
55 * processor.
56 */
57 #if PER_CPU_CONTROL_SIZE_APPROX > 1024
58 #define PER_CPU_CONTROL_SIZE_LOG2 11
59 #elif PER_CPU_CONTROL_SIZE_APPROX > 512
60 #define PER_CPU_CONTROL_SIZE_LOG2 10
61 #elif PER_CPU_CONTROL_SIZE_APPROX > 256
62 #define PER_CPU_CONTROL_SIZE_LOG2 9
63 #elif PER_CPU_CONTROL_SIZE_APPROX > 128
64 #define PER_CPU_CONTROL_SIZE_LOG2 8
65 #else
66 #define PER_CPU_CONTROL_SIZE_LOG2 7
67 #endif
68
69 #define PER_CPU_CONTROL_SIZE ( 1 << PER_CPU_CONTROL_SIZE_LOG2 )
70#endif
71
72#if !defined( ASM )
73
74struct Record_Control;
75
76struct _Thread_Control;
77
78struct Scheduler_Context;
79
93#if defined( RTEMS_SMP )
94
128typedef enum {
137 PER_CPU_STATE_INITIAL,
138
153 PER_CPU_STATE_READY_TO_START_MULTITASKING,
154
163 PER_CPU_STATE_REQUEST_START_MULTITASKING,
164
168 PER_CPU_STATE_UP,
169
173 PER_CPU_STATE_SHUTDOWN
174} Per_CPU_State;
175
176typedef void ( *Per_CPU_Job_handler )( void *arg );
177
184typedef struct {
188 Per_CPU_Job_handler handler;
189
193 void *arg;
194} Per_CPU_Job_context;
195
196/*
197 * Value for the Per_CPU_Job::done member to indicate that a job is done
198 * (handler was called on the target processor). Must not be a valid pointer
199 * value since it overlaps with the Per_CPU_Job::next member.
200 */
201#define PER_CPU_JOB_DONE 1
202
209typedef struct Per_CPU_Job {
210 union {
214 struct Per_CPU_Job *next;
215
222 Atomic_Ulong done;
223 };
224
228 const Per_CPU_Job_context *context;
229} Per_CPU_Job;
230
231#endif /* defined( RTEMS_SMP ) */
232
236 typedef struct {
237#if defined( RTEMS_PROFILING )
243 CPU_Counter_ticks thread_dispatch_disabled_instant;
244
249 CPU_Counter_ticks max_thread_dispatch_disabled_time;
250
258 CPU_Counter_ticks max_interrupt_time;
259
264 CPU_Counter_ticks max_interrupt_delay;
265
272 uint64_t thread_dispatch_disabled_count;
273
283 uint64_t total_thread_dispatch_disabled_time;
284
291 uint64_t interrupt_count;
292
301 uint64_t total_interrupt_time;
302#endif /* defined( RTEMS_PROFILING ) */
303} Per_CPU_Stats;
304
308 typedef enum {
316 PER_CPU_WATCHDOG_TICKS,
317
325 PER_CPU_WATCHDOG_REALTIME,
326
334 PER_CPU_WATCHDOG_MONOTONIC,
335
339 PER_CPU_WATCHDOG_COUNT
340} Per_CPU_Watchdog_index;
341
347 typedef struct Per_CPU_Control {
348 #if CPU_PER_CPU_CONTROL_SIZE > 0
352 CPU_Per_CPU_control cpu_per_cpu;
353 #endif
354
358 void *interrupt_stack_low;
359
363 void *interrupt_stack_high;
364
369 uint32_t isr_nest_level;
370
379 uint32_t isr_dispatch_disable;
380
385 volatile uint32_t thread_dispatch_disable_level;
386
400 volatile bool dispatch_necessary;
401
402 /*
403 * Ensure that the executing member is at least 4-byte aligned, see
404 * PER_CPU_OFFSET_EXECUTING. This is necessary on CPU ports with relaxed
405 * alignment restrictions, e.g. type alignment is less than the type size.
406 */
407 bool reserved_for_executing_alignment[ 3 ];
408
420 struct _Thread_Control *executing;
421
436 struct _Thread_Control *heir;
437
438#if defined(RTEMS_SMP)
439 CPU_Interrupt_frame Interrupt_frame;
440#endif
441
452 Timestamp_Control cpu_usage_timestamp;
453
457 struct {
461 ISR_LOCK_MEMBER( Lock )
462
463
467 uint64_t ticks;
468
474 Watchdog_Header Header[ PER_CPU_WATCHDOG_COUNT ];
475 } Watchdog;
476
477 #if defined( RTEMS_SMP )
481 ISR_lock_Control Lock;
482
490 ISR_lock_Context Lock_context;
491
497 Chain_Control Threads_in_need_for_help;
498
505 Atomic_Ulong message;
506
507 struct {
514 const struct _Scheduler_Control *control;
515
522 const struct Scheduler_Context *context;
523
528 struct _Thread_Control *idle_if_online_and_unused;
529 } Scheduler;
530
536 struct _Thread_Control *ancestor;
537
543 char *data;
544
552 Per_CPU_State state;
553
559 struct {
564 ISR_lock_Control Lock;
565
572 struct Per_CPU_Job *head;
573
582 struct Per_CPU_Job **tail;
583 } Jobs;
584
589 bool online;
590
595 bool boot;
596 #endif
597
598 struct Record_Control *record;
599
600 Per_CPU_Stats Stats;
601} Per_CPU_Control;
602
603#if defined( RTEMS_SMP )
604typedef struct {
605 Per_CPU_Control per_cpu;
606 char unused_space_for_cache_line_alignment
607 [ PER_CPU_CONTROL_SIZE - sizeof( Per_CPU_Control ) ];
608} Per_CPU_Control_envelope;
609#else
610 typedef struct {
611 Per_CPU_Control per_cpu;
612} Per_CPU_Control_envelope;
613#endif
614
620extern Per_CPU_Control_envelope _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT;
621
622#define _Per_CPU_Acquire( cpu, lock_context ) \
623 _ISR_lock_Acquire( &( cpu )->Lock, lock_context )
624
625#define _Per_CPU_Release( cpu, lock_context ) \
626 _ISR_lock_Release( &( cpu )->Lock, lock_context )
627
628/*
629 * If we get the current processor index in a context which allows thread
630 * dispatching, then we may already run on another processor right after the
631 * read instruction. There are very few cases in which this makes sense (here
632 * we can use _Per_CPU_Get_snapshot()). All other places must use
633 * _Per_CPU_Get() so that we can add checks for RTEMS_DEBUG.
634 */
635#if defined( _CPU_Get_current_per_CPU_control )
636 #define _Per_CPU_Get_snapshot() _CPU_Get_current_per_CPU_control()
637#else
638 #define _Per_CPU_Get_snapshot() \
639 ( &_Per_CPU_Information[ _SMP_Get_current_processor() ].per_cpu )
640#endif
641
642#if defined( RTEMS_SMP )
643static inline Per_CPU_Control *_Per_CPU_Get( void )
644{
645 Per_CPU_Control *cpu_self = _Per_CPU_Get_snapshot();
646
647 _Assert(
648 cpu_self->thread_dispatch_disable_level != 0 || _ISR_Get_level() != 0
649 );
650
651 return cpu_self;
652}
653#else
654#define _Per_CPU_Get() _Per_CPU_Get_snapshot()
655#endif
656
657static inline Per_CPU_Control *_Per_CPU_Get_by_index( uint32_t index )
658{
659 return &_Per_CPU_Information[ index ].per_cpu;
660}
661
662static inline uint32_t _Per_CPU_Get_index( const Per_CPU_Control *cpu )
663{
664#if defined(RTEMS_SMP)
665 const Per_CPU_Control_envelope *per_cpu_envelope =
666 ( const Per_CPU_Control_envelope * ) cpu;
667
668 return ( uint32_t ) ( per_cpu_envelope - &_Per_CPU_Information[ 0 ] );
669#else
670 (void) cpu;
671 return 0;
672#endif
673}
674
675static inline struct _Thread_Control *_Per_CPU_Get_executing(
676 const Per_CPU_Control *cpu
677)
678{
679 return cpu->executing;
680}
681
682static inline bool _Per_CPU_Is_processor_online(
683 const Per_CPU_Control *cpu
684)
685{
686#if defined( RTEMS_SMP )
687 return cpu->online;
688#else
689 (void) cpu;
690
691 return true;
692#endif
693}
694
695static inline bool _Per_CPU_Is_boot_processor(
696 const Per_CPU_Control *cpu
697)
698{
699#if defined( RTEMS_SMP )
700 return cpu->boot;
701#else
702 (void) cpu;
703
704 return true;
705#endif
706}
707
708 RTEMS_INLINE_ROUTINE void _Per_CPU_Acquire_all(
709 ISR_lock_Context *lock_context
710)
711{
712#if defined(RTEMS_SMP)
713 uint32_t cpu_max;
714 uint32_t cpu_index;
715 Per_CPU_Control *previous_cpu;
716
717 cpu_max = _SMP_Get_processor_maximum();
718 previous_cpu = _Per_CPU_Get_by_index( 0 );
719
720 _ISR_lock_ISR_disable( lock_context );
721 _Per_CPU_Acquire( previous_cpu, lock_context );
722
723 for ( cpu_index = 1 ; cpu_index < cpu_max ; ++cpu_index ) {
724 Per_CPU_Control *cpu;
725
726 cpu = _Per_CPU_Get_by_index( cpu_index );
727 _Per_CPU_Acquire( cpu, &previous_cpu->Lock_context );
728 previous_cpu = cpu;
729 }
730#else
731 _ISR_lock_ISR_disable( lock_context );
732#endif
733}
734
735 RTEMS_INLINE_ROUTINE void _Per_CPU_Release_all(
736 ISR_lock_Context *lock_context
737)
738{
739#if defined(RTEMS_SMP)
740 uint32_t cpu_max;
741 uint32_t cpu_index;
742 Per_CPU_Control *cpu;
743
744 cpu_max = _SMP_Get_processor_maximum();
745 cpu = _Per_CPU_Get_by_index( cpu_max - 1 );
746
747 for ( cpu_index = cpu_max - 1 ; cpu_index > 0 ; --cpu_index ) {
748 Per_CPU_Control *previous_cpu;
749
750 previous_cpu = _Per_CPU_Get_by_index( cpu_index - 1 );
751 _Per_CPU_Release( cpu, &previous_cpu->Lock_context );
752 cpu = previous_cpu;
753 }
754
755 _Per_CPU_Release( cpu, lock_context );
756 _ISR_lock_ISR_enable( lock_context );
757#else
758 _ISR_lock_ISR_enable( lock_context );
759#endif
760}
761
762#if defined( RTEMS_SMP )
763
769void _Per_CPU_Initialize(void);
770
771void _Per_CPU_State_change(
772 Per_CPU_Control *cpu,
773 Per_CPU_State new_state
774);
775
801bool _Per_CPU_State_wait_for_non_initial_state(
802 uint32_t cpu_index,
803 uint32_t timeout_in_ns
804);
805
811void _Per_CPU_Perform_jobs( Per_CPU_Control *cpu );
812
824void _Per_CPU_Add_job( Per_CPU_Control *cpu, Per_CPU_Job *job );
825
835void _Per_CPU_Wait_for_job(
836 const Per_CPU_Control *cpu,
837 const Per_CPU_Job *job
838);
839
840#endif /* defined( RTEMS_SMP ) */
841
842/*
843 * On a non SMP system, the _SMP_Get_current_processor() is defined to 0.
844 * Thus when built for non-SMP, there should be no performance penalty.
845 */
846#define _Thread_Dispatch_disable_level \
847 _Per_CPU_Get()->thread_dispatch_disable_level
848#define _Thread_Heir \
849 _Per_CPU_Get()->heir
850
851#if defined(_CPU_Get_thread_executing)
852#define _Thread_Executing \
853 _CPU_Get_thread_executing()
854#else
855#define _Thread_Executing \
856 _Per_CPU_Get_executing( _Per_CPU_Get() )
857#endif
858
859#define _ISR_Nest_level \
860 _Per_CPU_Get()->isr_nest_level
861#define _CPU_Interrupt_stack_low \
862 _Per_CPU_Get()->interrupt_stack_low
863#define _CPU_Interrupt_stack_high \
864 _Per_CPU_Get()->interrupt_stack_high
865#define _Thread_Dispatch_necessary \
866 _Per_CPU_Get()->dispatch_necessary
867
878 RTEMS_INLINE_ROUTINE struct _Thread_Control *_Thread_Get_executing( void )
879{
880 struct _Thread_Control *executing;
881
882 #if defined(RTEMS_SMP) && !defined(_CPU_Get_thread_executing)
883 ISR_Level level;
884
885 _ISR_Local_disable( level );
886 #endif
887
888 executing = _Thread_Executing;
889
890 #if defined(RTEMS_SMP) && !defined(_CPU_Get_thread_executing)
891 _ISR_Local_enable( level );
892 #endif
893
894 return executing;
895}
896
899#endif /* !defined( ASM ) */
900
901#if defined( ASM ) || defined( _RTEMS_PERCPU_DEFINE_OFFSETS )
902
903#define PER_CPU_INTERRUPT_STACK_LOW \
904 CPU_PER_CPU_CONTROL_SIZE
905#define PER_CPU_INTERRUPT_STACK_HIGH \
906 PER_CPU_INTERRUPT_STACK_LOW + CPU_SIZEOF_POINTER
907
908#define INTERRUPT_STACK_LOW \
909 (SYM(_Per_CPU_Information) + PER_CPU_INTERRUPT_STACK_LOW)
910#define INTERRUPT_STACK_HIGH \
911 (SYM(_Per_CPU_Information) + PER_CPU_INTERRUPT_STACK_HIGH)
912
913/*
914 * These are the offsets of the required elements in the per CPU table.
915 */
916#define PER_CPU_ISR_NEST_LEVEL \
917 PER_CPU_INTERRUPT_STACK_HIGH + CPU_SIZEOF_POINTER
918#define PER_CPU_ISR_DISPATCH_DISABLE \
919 PER_CPU_ISR_NEST_LEVEL + 4
920#define PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL \
921 PER_CPU_ISR_DISPATCH_DISABLE + 4
922#define PER_CPU_DISPATCH_NEEDED \
923 PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL + 4
924#define PER_CPU_OFFSET_EXECUTING \
925 PER_CPU_DISPATCH_NEEDED + 4
926#define PER_CPU_OFFSET_HEIR \
927 PER_CPU_OFFSET_EXECUTING + CPU_SIZEOF_POINTER
928#if defined(RTEMS_SMP)
929#define PER_CPU_INTERRUPT_FRAME_AREA \
930 PER_CPU_OFFSET_HEIR + CPU_SIZEOF_POINTER
931#endif
932
933#define THREAD_DISPATCH_DISABLE_LEVEL \
934 (SYM(_Per_CPU_Information) + PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL)
935#define ISR_NEST_LEVEL \
936 (SYM(_Per_CPU_Information) + PER_CPU_ISR_NEST_LEVEL)
937#define DISPATCH_NEEDED \
938 (SYM(_Per_CPU_Information) + PER_CPU_DISPATCH_NEEDED)
939
940#endif /* defined( ASM ) || defined( _RTEMS_PERCPU_DEFINE_OFFSETS ) */
941
942#ifdef __cplusplus
943}
944#endif
945
946#endif
947/* end of include file */
assert.h
Information for the Assert Handler.
watchdog.h
Constants and Structures Associated with Watchdog Timers.
CPU_STRUCTURE_ALIGNMENT
Per_CPU_Control_envelope _Per_CPU_Information[] CPU_STRUCTURE_ALIGNMENT
Set of Per CPU Core Information.
_Thread_Get_executing
RTEMS_INLINE_ROUTINE struct _Thread_Control * _Thread_Get_executing(void)
Returns the thread control block of the executing thread.
Definition: percpu.h:878
Per_CPU_Watchdog_index
Per_CPU_Watchdog_index
Per-CPU watchdog header index.
Definition: percpu.h:308
PER_CPU_WATCHDOG_MONOTONIC
@ PER_CPU_WATCHDOG_MONOTONIC
Index for monotonic clock per-CPU watchdog header.
Definition: percpu.h:334
PER_CPU_WATCHDOG_COUNT
@ PER_CPU_WATCHDOG_COUNT
Count of per-CPU watchdog headers.
Definition: percpu.h:339
PER_CPU_WATCHDOG_REALTIME
@ PER_CPU_WATCHDOG_REALTIME
Index for realtime clock per-CPU watchdog header.
Definition: percpu.h:325
PER_CPU_WATCHDOG_TICKS
@ PER_CPU_WATCHDOG_TICKS
Index for tick clock per-CPU watchdog header.
Definition: percpu.h:316
_Assert
#define _Assert(_e)
Assertion similar to assert() controlled via RTEMS_DEBUG instead of NDEBUG.
Definition: assert.h:100
RTEMS_INLINE_ROUTINE
#define RTEMS_INLINE_ROUTINE
Definition: basedefs.h:66
ISR_LOCK_MEMBER
#define ISR_LOCK_MEMBER(_designator)
Defines an ISR lock member.
Definition: isrlock.h:89
_ISR_lock_ISR_enable
#define _ISR_lock_ISR_enable(_context)
Restores the saved interrupt state of the ISR lock context.
Definition: isrlock.h:419
_ISR_lock_ISR_disable
#define _ISR_lock_ISR_disable(_context)
Disables interrupts and saves the previous interrupt state in the ISR lock context.
Definition: isrlock.h:398
_ISR_Local_disable
#define _ISR_Local_disable(_level)
Disables interrupts on this processor.
Definition: isrlevel.h:57
_ISR_Local_enable
#define _ISR_Local_enable(_level)
Enables interrupts on this processor.
Definition: isrlevel.h:74
_ISR_Get_level
#define _ISR_Get_level()
Return current interrupt level.
Definition: isrlevel.h:128
ISR_Level
uint32_t ISR_Level
Definition: isrlevel.h:41
Timestamp_Control
int64_t Timestamp_Control
Definition: timestamp.h:57
isrlock.h
ISR Locks.
chain.h
Chain Handler API.
smp.h
SuperCore SMP Support API.
CPU_Interrupt_frame
Interrupt stack frame (ISF).
Definition: cpu.h:191
CPU_Per_CPU_control
The CPU specific per-CPU control.
Definition: cpuimpl.h:54
ISR_lock_Context
Local ISR lock context for acquire and release pairs.
Definition: isrlock.h:65
ISR_lock_Control
ISR lock control.
Definition: isrlock.h:56
Per_CPU_Control_envelope
Definition: percpu.h:610
Per_CPU_Control
Per CPU Core Structure.
Definition: percpu.h:347
Per_CPU_Control::heir
struct _Thread_Control * heir
This is the heir thread for this processor.
Definition: percpu.h:436
Per_CPU_Control::isr_dispatch_disable
uint32_t isr_dispatch_disable
Indicates if an ISR thread dispatch is disabled.
Definition: percpu.h:379
Per_CPU_Control::interrupt_stack_high
void * interrupt_stack_high
The interrupt stack high address for this processor.
Definition: percpu.h:363
Per_CPU_Control::thread_dispatch_disable_level
volatile uint32_t thread_dispatch_disable_level
The thread dispatch critical section nesting counter which is used to prevent context switches at ino...
Definition: percpu.h:385
Per_CPU_Control::dispatch_necessary
volatile bool dispatch_necessary
This is set to true when this processor needs to run the thread dispatcher.
Definition: percpu.h:400
Per_CPU_Control::cpu_usage_timestamp
Timestamp_Control cpu_usage_timestamp
The CPU usage timestamp contains the time point of the last heir thread change or last CPU usage upda...
Definition: percpu.h:452
Per_CPU_Control::executing
struct _Thread_Control * executing
This is the thread executing on this processor.
Definition: percpu.h:420
Per_CPU_Control::isr_nest_level
uint32_t isr_nest_level
Definition: percpu.h:369
Per_CPU_Control::interrupt_stack_low
void * interrupt_stack_low
The interrupt stack low address for this processor.
Definition: percpu.h:358
Per_CPU_Stats
Per-CPU statistics.
Definition: percpu.h:236
Record_Control
Definition: record.h:44
Scheduler_Context
Scheduler context.
Definition: scheduler.h:252
Watchdog_Header
The watchdog header to manage scheduled watchdogs.
Definition: watchdog.h:71
_Scheduler_Control
Scheduler control.
Definition: scheduler.h:269
_Thread_Control
Definition: thread.h:732
control
Definition: intercom.c:74
message
Definition: media-server.c:33
timestamp.h
Helpers for Manipulating Timestamps.
Chain_Control
Definition: chain.h:86