In HPC, we may need to protect head node from unnecessary heavy process that may cause login problem for users. One of the solutions is by using cpulimit. We can create a cronjob to monitor all processes and set certain limit for them. This is how I usually did in CentOS/RHEL 7.x.
yum install cpulimit
- Create a script to monitor the process. The script below is a modified version of the script in this forum. You can modify inputs of the first 3 variables:
CPU_LIMIT, BLACK_PROCESSES_LIST, and WHITE_PROCESSES_LIST.
$ cat cpulimitmon
[sourcecode language=”bash”]
#!/bin/bash
# ==============================================================
# CPU limit daemon – set PID’s max. percentage CPU consumptions
# ==============================================================
# Variables
CPU_LIMIT=100 # Maximum percentage CPU consumption by each PID
BLACK_PROCESSES_LIST= # Limit only processes defined in this variable. If variable is empty (default) all violating processes are limited.
WHITE_PROCESSES_LIST= # Limit all processes except processes defined in this variable. If variable is empty (default) all violating processes are limited.
# Check if one of the variables BLACK_PROCESSES_LIST or WHITE_PROCESSES_LIST is defined.
if [[ -n "$BLACK_PROCESSES_LIST" && -n "$WHITE_PROCESSES_LIST" ]] ; then # If both variables are defined then error is produced.
echo "At least one or both of the variables BLACK_PROCESSES_LIST or WHITE_PROCESSES_LIST must be empty."
exit 1
elif [[ -n "$BLACK_PROCESSES_LIST" ]] ; then # If this variable is non-empty then set NEW_PIDS_COMMAND variable to bellow command
NEW_PIDS_COMMAND="top -b -n1 | grep -E ‘$BLACK_PROCESSES_LIST’ | gawk ‘\$9>=CPU_LIMIT {print \$1}’ CPU_LIMIT=$CPU_LIMIT"
elif [[ -n "$WHITE_PROCESSES_LIST" ]] ; then # If this variable is non-empty then set NEW_PIDS_COMMAND variable to bellow command
NEW_PIDS_COMMAND="top -b -n1 -c | gawk ‘NR>7’ | grep -E -v ‘$WHITE_PROCESSES_LIST’ | gawk ‘\$9>CPU_LIMIT {print \$1}’ CPU_LIMIT=$CPU_LIMIT"
else
NEW_PIDS_COMMAND="top -b -n1 | gawk ‘NR>7 && \$9>=CPU_LIMIT {print \$1}’ CPU_LIMIT=$CPU_LIMIT"
fi
# Search and limit violating PIDs
NEW_PIDS=$(eval "$NEW_PIDS_COMMAND") # Violating PIDs
LIMITED_PIDS=$(ps -eo args | gawk ‘$1=="cpulimit" {print $3}’) # Already limited PIDs
QUEUE_PIDS=$(comm -23 <(echo "$NEW_PIDS" | sort -u) <(echo "$LIMITED_PIDS" | sort -u) | grep -v ‘^$’) # PIDs in queue
NP=""
for i in $NEW_PIDS
do
NP="$NP $i"
done
LP=""
for i in $LIMITED_PIDS
do
LP="$LP $i"
done
QP=""
for i in $QUEUE_PIDS
do
QP="$QP $i"
done
echo "[`date +’%Y%m%d %T’`] NEWPIDS =$NP"
echo "[`date +’%Y%m%d %T’`] LIMPIDS =$LP"
echo "[`date +’%Y%m%d %T’`] QUEPIDS =$QP"
for i in $QUEUE_PIDS
do
echo "[`date +’%Y%m%d %T’`] set cpu limit to pid $i"
cpulimit -p $i -l $CPU_LIMIT -z & # Limit new violating processes
done
[/sourcecode]
- Set it as executable file.
$ chmod +x cpulimitmon
- Create a cronjob to execute the script periodically. The cronjob should be under root user.
$ crontab -e
* * * * * /path/to/cpulimitmon >> /var/log/cpulimitmon.log 2>&1