[PlanetCCRMA] guaranteed deadlock w/ 2.4.20-4.ll.acpi
Fernando Pablo Lopez-Lezcano
nando@ccrma.Stanford.EDU
Wed May 14 13:19:01 2003
--=-A6Y4B5oYk28cw6pE3eRI
Content-Type: text/plain
Content-Transfer-Encoding: 7bit
> > > I am running redhat 8.0 with kernel-2.4.20-4.ll.acpi, jack-0.7.x (tried
> > > 0.70.. and the freshly updated 0.71..) and ecasound. Filesystems are all
> > > ext3. Even when I run jack without the "-R" switch and as an unpriviliged
> > > user, using 2 simultaneous sessions of ecasound to record (ecasound -t:60
> > > -i:jack_alsa -G:jack,ecasound_rec1,streaming -o stdout) as root with or
> > > without the "-r" switch, my system will deadlock reliably and immeadiatly.
> >
> > Is this only happening with ecasound? (and only with two copies running
> > at the same time?) Or do you see the problem with other jack software as
> > well?
>
> no, I managed to crash it a couple of times before, while using a
> arecord and jackrec simultaneously. When that happened, jackrec and
> arecord were being run without privilieges, but jackd was running with
> the "-R" (realtime) switch
That is strange. You should not be able to run arecord and jackrec at
the same time. Jackd monopolizes the alsa device so arecord should find
it busy (and eventually timeout?).
When you run jackd with -R all jack clients get a thread that runs with
realtime priority (that is automatic). So in your case both jackd and
the audio thread of jackrec were running SCHED_FIFO.
Roger Larsson sent me a small app to monitor SCHED_FIFO processes and
downgrade them to normal schedulling when they start eating all the cpu
for a (programmable) time. It was very useful when I was having lock up
problems (in that case due to a problem in ext3). The machine would lock
up for 5 seconds and then magically I would get control again :-) I'm
adding the files as attachments to this email.
> > > It still responds to pings but nothing else. I am running with low-latency
> > > on. The system is remote, so it's sort of a pain to test out different
> > > configurations.
> >
> > Yuck. It sounds like an infinite loop on a SCHED_FIFO process (ie:
> > anything interrupt related may work but no processes get to be
> > schedulled at all other than the realtime process sucking up all cpu
> > cycles). But you mention that it also happens when you run as a normal
> > users with no special privileges so that should not be the problem.
>
> no, something has to be running with priviliges. In the case I
> described, jackd was running un-priviliged,
Meaning as root but without "-R"?
> but both ecasound sessions were running as root.
>
> > Of course, as you say, it could also be alsa...
>
> possibly. there is a fix regarding spinlocks that went into alsa CVS on
> April 26th. Don't know if that fix made it into 0.9.3 or not.
The current Planet CCRMA alsa packages are from CVS, dated April 9th...
> While my
> system is UP, doesn't preempt use spinlocks? Does 2.4.20-4.ll.acpi have
> preempt enabled?
Yes.
-- Fernando
--=-A6Y4B5oYk28cw6pE3eRI
Content-Disposition: attachment; filename=Makefile
Content-Transfer-Encoding: quoted-printable
Content-Type: text/x-makefile; name=Makefile; charset=ISO-8859-1
CFLAGS=3D-g
all: rt rt_monitor
rt: rt.c
rt_monitor: rt_monitor.c
--=-A6Y4B5oYk28cw6pE3eRI
Content-Disposition: attachment; filename=rt.c
Content-Transfer-Encoding: quoted-printable
Content-Type: text/x-c; name=rt.c; charset=ISO-8859-1
/* RT user.
Copyright (c) 2002 Roger Larsson <roger.larsson@norran.net>
This program is free software; you can redistribute it and/or
modify it under the terms of version 2 of the GNU General Public
License as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
Thanks to autor of KSysGuard Chris Schlaeger for borrowed code...
*/
#include <sys/types.h>
#include <sched.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <stdlib.h>
struct request
{
pid_t pid;
char _filler[32];
};
int isRT(pid_t pid)
{
int sched_class =3D sched_getscheduler( pid);
if (sched_class =3D=3D -1) {
fprintf(stderr, "Pid %d Exited?\n", pid);
return 0;
}
return sched_class !=3D SCHED_OTHER;
}
int main(int argc, char *argv[])
{
struct request request;
FILE *reqf;
int done=3D0, loops =3D 0;
request.pid =3D getpid();
while (!done) {
switch (getopt(argc, argv, "?c:p:q:" )) {
case 'c':
loops =3D atoi(optarg);
break;
case 'p':
request.pid =3D atoi(optarg);
printf("pid %d\n", request.pid);
break;
case 'q':
request.pid =3D atoi(optarg);
printf("pid %d - %s\n", request.pid, isRT(request.pid) ? "is RT" : "i=
s NOT RT");
return;
case '?':
printf("%s: [-c|-p pid]\n", argv[0]);
printf("\t-c loops\tcheck monitor function by looping\n");
printf("\t Note: I need 9000000 at 933MHz, start low and move up\n"=
);
printf("\t-p pid\trequest on behalf of other process\n");
printf("\t-q pid\tquery if process is RT\n");
return 1;
case -1:
// No more options
done =3D 1;
break;
}
}
printf("As long as no monitor runs, execution will sleep here...\n");
reqf =3D fopen("/var/named/rt-request", "w");
if (reqf =3D=3D NULL) {
perror("fopen");
return errno;
}
printf("policy %d\n", sched_getscheduler(request.pid));
printf("Wait...\n"); // Has to be here to get some chance to display...
fwrite(&request, 32, 1, reqf);
fclose(reqf); // important! (maybe flush?)
// Wait until RT monitor raised prio
while (sched_getscheduler(request.pid) =3D=3D 0) {
}
printf("policy %d\n", sched_getscheduler(request.pid));
// well behaved
if (request.pid =3D=3D getpid() && loops > 0) {
printf("\nsleep for 3 seconds then start with a\n");
printf("busy wait for %d loops (or until prio reduced)\n", loops);
printf(" move your mouse!\n");
sleep(3);
while (--loops > 0 && sched_getscheduler(request.pid) !=3D 0) {
// someone did listen to my request...
// assume monitor is running
}
if (loops =3D=3D 0)
printf(" - normal loop finish, to short loop?\n");
else
printf(" - monitor works! (priority got reduced)\n");
}
return 0;
}
--=-A6Y4B5oYk28cw6pE3eRI
Content-Disposition: attachment; filename=rt_monitor.c
Content-Transfer-Encoding: quoted-printable
Content-Type: text/x-c; name=rt_monitor.c; charset=ISO-8859-1
/* RT monitor.
Copyright (c) 2002 Roger Larsson <roger.larsson@norran.net>
This program is free software; you can redistribute it and/or
modify it under the terms of version 2 of the GNU General Public
License as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
Thanks to autor of KSysGuard Chris Schlaeger for borrowed code...
*/
#include <sys/types.h>
#include <sched.h>
#include <stdio.h>
#include <dirent.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <ctype.h>
#include <sys/time.h>
#include <stdlib.h>
#include <signal.h>
#include <sys/resource.h>
int set_normal_priority(pid_t pid);
int isRT(pid_t pid)
{
int sched_class =3D sched_getscheduler( pid);
if (sched_class =3D=3D -1) {
fprintf(stderr, "Pid %d Exited?\n", pid);
return 0;
}
return sched_class !=3D SCHED_OTHER;
}
struct rt_process_info
{
/* This flag is set for all found processes at the beginning of the
* process list update. Processes that do not have this flag set will
* be assumed dead and removed from the list. The flag is cleared after
* each list update. */
int alive;
int centStamp;
char state;
pid_t pid;
pid_t ppid;
gid_t gid;
unsigned long userTime;
unsigned long sysTime;
unsigned long vmSize; // enough?
long vmRss; // enough?
unsigned long eip;
float sysLoad;
float userLoad;
float cpu_usage;
};
#define MAX_RT_PROCESSES 200
struct rt_process_info rt_process[MAX_RT_PROCESSES]; /* pid & alive =3D=3D =
0 */
struct rt_process_info *find_process(pid_t pid)
{
unsigned ix;
for (ix =3D 0; ix < MAX_RT_PROCESSES; ix++)
{
if (rt_process[ix].pid =3D=3D pid) {
rt_process[ix].alive =3D 1;
return &rt_process[ix];
}
}
return NULL;
}
struct rt_process_info *new_process(pid_t pid)
{
unsigned ix;
for (ix =3D 0; ix < MAX_RT_PROCESSES; ix++)
{
if (rt_process[ix].pid =3D=3D 0) {
rt_process[ix].pid =3D pid;
rt_process[ix].alive =3D 2;
return &rt_process[ix];
}
}
return NULL;
}
float cpu_usage(struct rt_process_info *ps)
{
#define BUFSIZE 1024
char buf[BUFSIZE];
FILE *fd;
char status;
unsigned int userTime, sysTime;
snprintf(buf, BUFSIZE - 1, "/proc/%d/stat", ps->pid);
buf[BUFSIZE - 1] =3D '\0';
if ((fd =3D fopen(buf, "r")) =3D=3D 0)
return (-1);
if (fscanf(fd, "%*d %*s %c %d %d %*d %*d %*d " // state, ppid, pgid
"%*lu %*lu %*lu %*lu %*lu %lu %lu " // utime, stime
"%*ld %*ld %*ld %*ld %*ld %*ld %lu %ld " // vmSize, vmRss
"%*lu %*lu %*lu %*lu %*lu %lu", // eip
&ps->state, (int*) &ps->ppid, (int*) &ps->gid,
&userTime, &sysTime, &ps->vmSize,
&ps->vmRss, &ps->eip) !=3D 8) {
perror("fscanf");
fclose(fd);
return (-1);
}
if (fclose(fd))
return (-1);
{
unsigned int newCentStamp;
int timeDiff, userDiff, sysDiff;
struct timeval tv;
gettimeofday(&tv, 0);
newCentStamp =3D tv.tv_sec * 100 + tv.tv_usec / 10000;
// calculate load
if (ps->alive =3D=3D 2)
ps->sysLoad =3D ps->userLoad =3D 0.0f; /* can't give relieable number =
at the moment... */
else {
timeDiff =3D (int)(newCentStamp - ps->centStamp);
userDiff =3D userTime - ps->userTime;
sysDiff =3D sysTime - ps->sysTime;
=09
if ((timeDiff > 0) && (userDiff >=3D 0) && (sysDiff >=3D 0)) /* protec=
t from bad data */
{
ps->userLoad =3D ((double) userDiff / timeDiff) * 100.0;
ps->sysLoad =3D ((double) sysDiff / timeDiff) * 100.0;
}
else
ps->sysLoad =3D ps->userLoad =3D 0.0;
}
// update fields
ps->centStamp =3D newCentStamp;
ps->userTime =3D userTime;
ps->sysTime =3D sysTime;
}
=09
ps->cpu_usage =3D ps->userLoad + ps->sysLoad;
return ps->cpu_usage;
}
float process_cpu_usage(pid_t pid)
{
struct rt_process_info *process =3D find_process(pid);
float cpu_use;
if (process =3D=3D NULL)
{
// New process, but not RT - uninteresting!
if (!isRT(pid))
return 0.0f;
process =3D new_process(pid);
if (process =3D=3D NULL) {
// to many RT (or previous) processes!
// this process is new - assume a DOS attack
printf("DOS attack? kill process %d\n", pid);
if (!kill(pid, SIGKILL))
perror("kill");
return 0.0f;
}
=09
process->alive =3D 2; /* mark process new */
}
// Account current RT processes AND old ones!
cpu_use =3D cpu_usage(process);
process->alive =3D 1;
return cpu_use;
}
/* process reading code from ksysguard */
float cpu_rt_usage(struct rt_process_info **rt_list_head)
{
// Watch out for SMP effects...
=09
float result =3D 0.0f;
pid_t myself =3D getpid();
DIR* dir;
struct dirent* entry;
/* read in current process list via the /proc filesystem entry */
if ((dir =3D opendir("/proc")) =3D=3D NULL)
{
perror("Cannot open directory \'/proc\'!\n"
"The kernel needs to be compiled with support\n"
"for /proc filesystem enabled!\n");
return 0;
}
=09
// for all processes
while ((entry =3D readdir(dir)))
{
if (isdigit(entry->d_name[0]))
{
pid_t pid;
pid =3D atoi(entry->d_name);
=09
if (pid !=3D myself) {
result +=3D process_cpu_usage(pid);
}
}
}
closedir(dir);
=09
return result;
}
void gc_rt_processes()
{
unsigned ix;
printf("C:");
for (ix =3D 0; ix < MAX_RT_PROCESSES; ix++)
{
struct rt_process_info *rt_examine =3D &rt_process[ix];
if (rt_examine->alive)
{
rt_examine->alive =3D 0;
}
else if (rt_examine->pid)
{
printf("%d ", rt_examine->pid);
rt_examine->pid =3D 0; /* delete it! */
}
}
}
int set_me_realtime(void)
{
struct sched_param schp;
/*
* set the process to realtime privs
*/
memset(&schp, 0, sizeof(schp));
schp.sched_priority =3D sched_get_priority_max(SCHED_FIFO);
if (sched_setscheduler(0, SCHED_FIFO, &schp) !=3D 0) {
perror("sched_setscheduler");
return -1;
}
if(mlockall(MCL_CURRENT|MCL_FUTURE))
{
perror("mlockall() failed, exiting. mlock");
return -1;
}
return 0;
}
int set_realtime_priority(pid_t pid)
{
struct sched_param schp;
/*
* set the process to realtime privs
*/
printf("Attempt to set realtime for pid %d ", pid);
if (pid =3D=3D 0 || pid =3D=3D getpid()) {
printf("- ignored! (that is me)\n");
return -1;
}
memset(&schp, 0, sizeof(schp));
schp.sched_priority =3D sched_get_priority_min(SCHED_FIFO);
if (sched_setscheduler(pid, SCHED_FIFO, &schp) !=3D 0) {
printf("- failed!\n");
perror("sched_setscheduler");
return -1;
}
printf("- done!\n");
(void)process_cpu_usage(pid); // Add to RT process array ASAP
return 0;
}
int set_normal_priority(pid_t pid)
{
struct sched_param schp;
if (!isRT(pid))
return 0;
/*
* set the process to normal privs, most nice
*/
memset(&schp, 0, sizeof(schp));
schp.sched_priority =3D 0;
if (pid =3D=3D 0 || pid =3D=3D getpid()) {
printf("- ignored! (that is me)\n");
return -1;
}
if (sched_setscheduler(pid, SCHED_OTHER, &schp) !=3D 0) {
printf("- failed!\n");
perror("sched_setscheduler");
return -1;
}
if (setpriority(PRIO_PROCESS, pid, 20) !=3D 0) {
printf("- failed!\n");
perror("setpriority");
return -1;
}
return 0;
}
void set_normal_priority_all()
{
unsigned ix;
printf("Removing RT scheduling from processes:\n");
for (ix =3D 0; ix < MAX_RT_PROCESSES; ix++)
{
struct rt_process_info *process_info =3D &rt_process[ix];
if (process_info->pid) {
if (process_info->state !=3D 'S')
printf("%d%c.%lx ", process_info->pid, process_info->state, process_info-=
>eip);
else
printf("%d ", process_info->pid);
set_normal_priority(process_info->pid);
}
}
printf("\n");
}
struct request
{
pid_t pid;
char _filler[32];
};
=09
#define REQUEST_SIZE 32
int poll_request(int reqfd)
{
// Be VERY careful not to
// * block here...
// * get buffer overruns...
static int remaining =3D REQUEST_SIZE;
static struct request request;
char *next =3D ((char *)&request + REQUEST_SIZE - remaining);
int ret =3D read(reqfd, (void *)next, remaining);
if (ret =3D=3D -1) {
perror("read");
return 0;
}
remaining -=3D ret;
if (remaining =3D=3D 0) {
remaining =3D REQUEST_SIZE;
if (request.pid =3D=3D 0 || request.pid =3D=3D getpid()) {
fputs("attempt to forge the monitor\n", stderr);
return 0;
}
set_realtime_priority(request.pid);
return 1;
}
return 0;
}
#define RTREQUEST_FILE "/var/named/rt-request"=20
int main(int argc, char * argv[])
{
struct rt_process_info *rt_list =3D NULL;
int reqfd =3D open(RTREQUEST_FILE, O_RDONLY | O_NONBLOCK | O_NDELAY);
if (reqfd =3D=3D -1) {
perror("open " RTREQUEST_FILE);
fputs("have you created it? use 'mkfifo -m 622 " RTREQUEST_FILE "'\n", std=
err);
exit(1);
}
=20
// monitor process runs with realtime prio
set_me_realtime();
#define MIN_IDLE 10
#define MAX_RT_USAGE 70
while (1) {
gc_rt_processes();
poll_request(reqfd);
=09
if (cpu_rt_usage(&rt_list) > MAX_RT_USAGE) {
printf("Total CPU RT usage above MAX_RT_USAGE\n");
// build process trees from rt_list
// decide which tree to reduce to normal prio class
// (assume only one for simplicitly...)
=20
// reduce all processes in that tree
set_normal_priority_all();
// (may use nice to simulate prio levels)
// log a message
}
sleep(2);
}
=20
// process exiting - free elements on rt_list...
//free_rt_list(rt_list);
}
--=-A6Y4B5oYk28cw6pE3eRI--