From 043595ca0a779042b200882c888ae4bb881ddfda Mon Sep 17 00:00:00 2001 From: till straumann Date: Mon, 17 Aug 2020 14:18:51 +0200 Subject: [PATCH] Added test for mutex priority inheritance --- src/libCom/test/Makefile | 5 + .../test/epicsMutexPriorityInversionTest.cc | 248 ++++++++++++++++++ .../test/epicsMutexPriorityInversionTest.plt | 10 + 3 files changed, 263 insertions(+) create mode 100644 src/libCom/test/epicsMutexPriorityInversionTest.cc create mode 100644 src/libCom/test/epicsMutexPriorityInversionTest.plt diff --git a/src/libCom/test/Makefile b/src/libCom/test/Makefile index 6239e0b41..927373f7e 100644 --- a/src/libCom/test/Makefile +++ b/src/libCom/test/Makefile @@ -197,5 +197,10 @@ TESTPROD_HOST += cvtFastPerform cvtFastPerform_SRCS += cvtFastPerform.cpp testHarness_SRCS += cvtFastPerform.cpp +TESTPROD_HOST_Linux += epicsMutexPriorityInversionTest +epicsMutexPriorityInversionTest_SRCS += epicsMutexPriorityInversionTest.cc +epicsMutexPriorityInversionTest_SYS_LIBS_Linux += pthread +TESTSCRIPTS_HOST_Linux+=epicsMutexPriorityInversionTest.t + include $(TOP)/configure/RULES diff --git a/src/libCom/test/epicsMutexPriorityInversionTest.cc b/src/libCom/test/epicsMutexPriorityInversionTest.cc new file mode 100644 index 000000000..7445ed1c9 --- /dev/null +++ b/src/libCom/test/epicsMutexPriorityInversionTest.cc @@ -0,0 +1,248 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define THE_CPU 0 + +#define SPIN_SECS 3000000 + +typedef struct ThreadArg_ { + epicsEventId sync; + epicsEventId done; + epicsMutexId mtx; + volatile unsigned long end; + unsigned long lim; + int pri[3]; +} ThreadArg; + +static unsigned long getClockUs() +{ +struct timespec now; + if ( clock_gettime( CLOCK_MONOTONIC, &now ) ) { + testAbort("clock_gettime(CLOCK_MONOTONIC) failed"); + } + return ((unsigned long)now.tv_sec)*1000000 + ((unsigned long)now.tv_nsec)/1000; +} + +/* + * Set affinity of executing thread to 'cpu' + */ +static void setThreadAffinity(int cpu) +{ +cpu_set_t cset; + + CPU_ZERO( &cset ); + CPU_SET( cpu, &cset ); + testOk1 ( 0 == pthread_setaffinity_np( pthread_self(), sizeof(cset), &cset ) ); +} + +/* + * Ensure only 'cpu' is set in executing thread's affinity mask + */ +static void checkAffinity(int cpu) +{ +cpu_set_t cset; + + if ( pthread_getaffinity_np( pthread_self(), sizeof(cset), &cset) ) { + testFail("pthread_getaffinity_np FAILED"); + return; + } + testOk ( 1 == CPU_COUNT( &cset ) && CPU_ISSET( cpu, &cset ), "Checking CPU affinity mask" ); +} + + +/* + * Make sure executing thread uses SCHED_FIFO and + * store its priority a->pri[idx] + * + * RETURNS: 0 if SCHED_FIFO engaged, nonzero otherwise. + */ +static int checkThreadPri(ThreadArg *a, unsigned idx) +{ +int pol; +struct sched_param p; + if ( pthread_getschedparam( pthread_self(), &pol, &p ) ) { + testFail("pthread_getschedparam FAILED"); + return 1; + } + if ( a && idx < sizeof(a->pri)/sizeof(a->pri[0]) ) { + a->pri[idx] = p.sched_priority; + } + testOk1( SCHED_FIFO == pol ); + return (SCHED_FIFO != pol); +} + + +/* + * Low-priority thread. + * + * Lock mutex and signal to the medium-priority thread + * that it may proceed. + * + * Since all three (high-, medium- and low-priority threads) + * execute on the same CPU (via affinity) the medium- + * priority thread will then take over the CPU and prevent + * (unless priority-inheritance is enabled, that is) + * the low-priority thread from continueing on to unlock + * the mutex. + */ +static void loPriThread(void *parm) +{ +ThreadArg *a = (ThreadArg*)parm; + + checkAffinity( THE_CPU ); + + checkThreadPri( a, 0 ); + + epicsMutexMustLock( a->mtx ); + + epicsEventSignal( a->sync ); + + /* medium-priority thread takes over CPU and spins + * while the high-priority thread waits for the mutex. + * With priority-inheritance enabled this thread's + * priority will be increased so that the medium- + * priority thread can be preempted and we proceed + * to release the mutex. + */ + + epicsMutexUnlock( a->mtx ); +} + +static void hiPriThread(void *parm) +{ +ThreadArg *a = (ThreadArg*)parm; + + /* Try to get the mutex */ + epicsMutexMustLock( a->mtx ); + /* Record the time when we obtained the mutex */ + a->end = getClockUs(); + epicsMutexUnlock( a->mtx ); + /* Tell the main thread that the test done */ + epicsEventSignal( a->done ); +} + +static void miPriThread(void *parm) +{ +ThreadArg *a = (ThreadArg*)parm; + + /* Basic checks: + * - affinity must be set to use single CPU for all threads + * - SCHED_FIFO must be in effect + */ + checkAffinity( THE_CPU ); + checkThreadPri( a, 1 ); + + /* Create the low-priority thread. */ + epicsThreadMustCreate("testLo", + a->pri[0], + epicsThreadGetStackSize( epicsThreadStackMedium ), + loPriThread, + a); + + /* Wait until low-priority thread has taken the mutex */ + epicsEventMustWait( a->sync ); + + /* Compute the end-time for our spinning loop */ + a->lim = getClockUs() + SPIN_SECS; + a->end = 0; + + /* Create the high-priority thread. The hiPri thread will + * block for the mutex and + * if priority-inheritance is available: + * increase the low-priority thread's priority temporarily + * so it can proceed to release the mutex and hand it to + * the high-priority thread. + * if priority-inheritance is not available: + * the high-priority thread will have to wait until we are + * done spinning and the low-priority thread is scheduled + * again. + */ + epicsThreadMustCreate("testHi", + a->pri[2], + epicsThreadGetStackSize( epicsThreadStackMedium ), + hiPriThread, + a); + + /* Spin for some time; the goal is hogging the CPU and thus preventing + * the low-priority thread from being scheduled. + * If priority-inheritance is enabled then the low-priority thread's + * priority is temporarily increased so that we can be preempted. This + * then causes the high-priority thread to record the 'end' time which + * tells us that we can terminate early... + */ + while ( 0 == a->end && getClockUs() < a->lim ) + /* spin */; + + /* w/o priority-inheritance the low-priority thread may proceed at + * this point and release the mutex to the high-priority thread. + */ +} + + +#define NUM_TESTS 8 + +MAIN(epicsMutexPriorityInversionTest) +{ +ThreadArg a; +long hiPriStalledTimeUs; +struct sched_param p_pri; + + /* This happens too late - i.e., after initialization of libCom + * user must set in the environment... + epicsEnvSet("EPICS_MUTEX_USE_PRIORITY_INHERITANCE","YES"); + */ + + a.mtx = epicsMutexMustCreate(); + a.sync = epicsEventMustCreate( epicsEventEmpty ); + a.done = epicsEventMustCreate( epicsEventEmpty ); + a.pri[0] = epicsThreadPriorityLow; + a.pri[1] = epicsThreadPriorityMedium; + a.pri[2] = epicsThreadPriorityHigh; + + testPlan(NUM_TESTS); + + p_pri.sched_priority = sched_get_priority_min( SCHED_FIFO ); + if ( sched_setscheduler( 0, SCHED_FIFO, &p_pri ) ) { + testDiag("SCHED_FIFO not engaged - maybe you need to be root to run this test?"); + testFail("sched_setscheduler(SCHED_FIFO)"); + testSkip( NUM_TESTS - 1, "SCHED_FIFO not engaged" ); + return testDone(); + } else { + testPass("SCHED_FIFO can be used"); + } + + setThreadAffinity( THE_CPU ); + /* created threads should inherit CPU affinity mask */ + + epicsThreadMustCreate("testMi", + a.pri[1], + epicsThreadGetStackSize( epicsThreadStackMedium ), + miPriThread, + &a); + + epicsEventMustWait( a.done ); + + testOk1( (a.pri[0] < a.pri[1]) && (a.pri[1] < a.pri[2]) ); + + hiPriStalledTimeUs = a.end - (a.lim - SPIN_SECS); + + testDiag("High-priority thread stalled for %li us\n", hiPriStalledTimeUs); + testOk1( hiPriStalledTimeUs < 200 ); + + epicsEventDestroy( a.done ); + epicsEventDestroy( a.sync ); + epicsMutexDestroy( a.mtx ); + + return testDone(); +} diff --git a/src/libCom/test/epicsMutexPriorityInversionTest.plt b/src/libCom/test/epicsMutexPriorityInversionTest.plt new file mode 100644 index 000000000..2f6e2c591 --- /dev/null +++ b/src/libCom/test/epicsMutexPriorityInversionTest.plt @@ -0,0 +1,10 @@ +#!/usr/bin/perl + +use strict; +my $prog = "./$0"; +$prog =~ s/\.t$//; + +$ENV{EPICS_MUTEX_USE_PRIORITY_INHERITANCE} = "YES"; + +$ENV{HARNESS_ACTIVE} = 1 if scalar @ARGV && shift eq '-tap'; +exec "$prog" or die "Can't run $prog: $!\n";