I have taken delivery of two identically specced CentOS/cPanel servers, showing the same behaviour of a resting load average of 1.30, 1.21, 1.16 and yet the CPU is sitting 100% idle.
Hardware:
Xeon(R) CPU E3-1270
4GB RAM
Behavior:-
top shows CPU 99.9% idle
virtually no disk IO
Some command output :-
uname -a
Linux server.myserver.com 2.6.18-308.4.1.el5PAE #1 SMP Tue Apr 17 17:47:38 EDT 2012 i686 i686 i386 GNU/Linux
top
top - 10:37:50 up 1:47, 1 user, load average: 1.28, 1.20, 1.17
Tasks: 199 total, 1 running, 198 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.0%us, 0.0%sy, 0.0%ni, 99.9%id, 0.1%wa, 0.0%hi, 0.0%si, 0.0%st
Mem: 4125104k total, 438764k used, 3686340k free, 25788k buffers
Swap: 2096440k total, 0k used, 2096440k free, 291080k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
1 root 15 0 2160 640 552 S 0.0 0.0 0:00.89 init
2 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/0
3 root 34 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/0
4 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/0
5 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/1
6 root 34 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/1
7 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/1
8 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/2
9 root 35 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/2
10 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/2
11 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/3
12 root 34 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/3
13 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/3
14 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/4
15 root 34 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/4
16 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/4
17 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/5
18 root 38 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/5
19 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/5
20 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/6
21 root 34 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/6
22 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/6
23 root RT -5 0 0 0 S 0.0 0.0 0:00.00 migration/7
24 root 34 19 0 0 0 S 0.0 0.0 0:00.00 ksoftirqd/7
25 root RT -5 0 0 0 S 0.0 0.0 0:00.00 watchdog/7
26 root 10 -5 0 0 0 S 0.0 0.0 0:06.42 events/0
27 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/1
28 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/2
29 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/3
30 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/4
31 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/5
32 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/6
33 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 events/7
34 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khelper
35 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kthread
45 root 13 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/0
46 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/1
47 root 14 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/2
48 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/3
49 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/4
50 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/5
51 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/6
52 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kblockd/7
53 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 kacpid
189 root 11 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/0
190 root 11 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/1
191 root 12 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/2
192 root 12 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/3
193 root 13 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/4
194 root 13 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/5
195 root 14 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/6
196 root 14 -5 0 0 0 S 0.0 0.0 0:00.00 cqueue/7
199 root 10 -5 0 0 0 S 0.0 0.0 0:00.00 khubd
ps axf
PID TTY STAT TIME COMMAND
1 ? Ss 0:00 init [3]
2 ? S< 0:00 [migration/0]
3 ? SN 0:00 [ksoftirqd/0]
4 ? S< 0:00 [watchdog/0]
5 ? S< 0:00 [migration/1]
6 ? SN 0:00 [ksoftirqd/1]
7 ? S< 0:00 [watchdog/1]
8 ? S< 0:00 [migration/2]
9 ? SN 0:00 [ksoftirqd/2]
10 ? S< 0:00 [watchdog/2]
11 ? S< 0:00 [migration/3]
12 ? SN 0:00 [ksoftirqd/3]
13 ? S< 0:00 [watchdog/3]
14 ? S< 0:00 [migration/4]
15 ? SN 0:00 [ksoftirqd/4]
16 ? S< 0:00 [watchdog/4]
17 ? S< 0:00 [migration/5]
18 ? SN 0:00 [ksoftirqd/5]
19 ? S< 0:00 [watchdog/5]
20 ? S< 0:00 [migration/6]
21 ? SN 0:00 [ksoftirqd/6]
22 ? S< 0:00 [watchdog/6]
23 ? S< 0:00 [migration/7]
24 ? SN 0:00 [ksoftirqd/7]
25 ? S< 0:00 [watchdog/7]
26 ? S< 0:06 [events/0]
27 ? S< 0:00 [events/1]
28 ? S< 0:00 [events/2]
29 ? S< 0:00 [events/3]
30 ? S< 0:00 [events/4]
31 ? S< 0:00 [events/5]
32 ? S< 0:00 [events/6]
33 ? S< 0:00 [events/7]
34 ? S< 0:00 [khelper]
35 ? S< 0:00 [kthread]
45 ? S< 0:00 \_ [kblockd/0]
46 ? S< 0:00 \_ [kblockd/1]
47 ? S< 0:00 \_ [kblockd/2]
48 ? S< 0:00 \_ [kblockd/3]
49 ? S< 0:00 \_ [kblockd/4]
50 ? S< 0:00 \_ [kblockd/5]
51 ? S< 0:00 \_ [kblockd/6]
52 ? S< 0:00 \_ [kblockd/7]
53 ? S< 0:00 \_ [kacpid]
189 ? S< 0:00 \_ [cqueue/0]
190 ? S< 0:00 \_ [cqueue/1]
191 ? S< 0:00 \_ [cqueue/2]
192 ? S< 0:00 \_ [cqueue/3]
193 ? S< 0:00 \_ [cqueue/4]
194 ? S< 0:00 \_ [cqueue/5]
195 ? S< 0:00 \_ [cqueue/6]
196 ? S< 0:00 \_ [cqueue/7]
199 ? S< 0:00 \_ [khubd]
201 ? S< 0:00 \_ [kseriod]
301 ? S 0:00 \_ [khungtaskd]
302 ? S 0:00 \_ [pdflush]
303 ? S 0:00 \_ [pdflush]
304 ? S< 0:00 \_ [kswapd0]
305 ? S< 0:00 \_ [aio/0]
306 ? S< 0:00 \_ [aio/1]
307 ? S< 0:00 \_ [aio/2]
308 ? S< 0:00 \_ [aio/3]
309 ? S< 0:00 \_ [aio/4]
310 ? S< 0:00 \_ [aio/5]
311 ? S< 0:00 \_ [aio/6]
312 ? S< 0:00 \_ [aio/7]
472 ? S< 0:00 \_ [kpsmoused]
551 ? S< 0:00 \_ [ata/0]
552 ? S< 0:00 \_ [ata/1]
553 ? S< 0:00 \_ [ata/2]
554 ? S< 0:00 \_ [ata/3]
555 ? S< 0:00 \_ [ata/4]
556 ? S< 0:00 \_ [ata/5]
557 ? S< 0:00 \_ [ata/6]
558 ? S< 0:00 \_ [ata/7]
559 ? S< 0:00 \_ [ata_aux]
569 ? S< 0:00 \_ [scsi_eh_0]
570 ? S< 0:00 \_ [scsi_eh_1]
571 ? S< 0:00 \_ [scsi_eh_2]
572 ? S< 0:00 \_ [scsi_eh_3]
573 ? S< 0:00 \_ [scsi_eh_4]
574 ? S< 0:00 \_ [scsi_eh_5]
593 ? S< 0:00 \_ [kstriped]
630 ? S< 0:00 \_ [kjournald]
655 ? S< 0:00 \_ [kauditd]
1860 ? S< 0:00 \_ [kmpathd/0]
1861 ? S< 0:00 \_ [kmpathd/1]
1862 ? S< 0:00 \_ [kmpathd/2]
1863 ? S< 0:00 \_ [kmpathd/3]
1864 ? S< 0:00 \_ [kmpathd/4]
1865 ? S< 0:00 \_ [kmpathd/5]
1866 ? S< 0:00 \_ [kmpathd/6]
1867 ? S< 0:00 \_ [kmpathd/7]
1868 ? S< 0:00 \_ [kmpath_handlerd]
1902 ? S< 0:00 \_ [kjournald]
1904 ? S< 0:00 \_ [kjournald]
1906 ? S< 0:00 \_ [kjournald]
1908 ? S< 0:00 \_ [kjournald]
1910 ? S< 0:00 \_ [kjournald]
2184 ? S< 0:00 \_ [iscsi_eh]
2288 ? S< 0:00 \_ [cnic_wq]
2298 ? S< 0:00 \_ [bnx2i_thread/0]
2299 ? S< 0:00 \_ [bnx2i_thread/1]
2300 ? S< 0:00 \_ [bnx2i_thread/2]
2301 ? S< 0:00 \_ [bnx2i_thread/3]
2302 ? S< 0:00 \_ [bnx2i_thread/4]
2303 ? S< 0:00 \_ [bnx2i_thread/5]
2304 ? S< 0:00 \_ [bnx2i_thread/6]
2305 ? S< 0:00 \_ [bnx2i_thread/7]
2330 ? S< 0:00 \_ [ib_addr]
2359 ? S< 0:00 \_ [ib_mcast]
2360 ? S< 0:00 \_ [ib_inform]
2361 ? S< 0:00 \_ [local_sa]
2371 ? S< 0:00 \_ [iw_cm_wq]
2381 ? S< 0:00 \_ [ib_cm/0]
2382 ? S< 0:00 \_ [ib_cm/1]
2383 ? S< 0:00 \_ [ib_cm/2]
2384 ? S< 0:00 \_ [ib_cm/3]
2385 ? S< 0:00 \_ [ib_cm/4]
2386 ? S< 0:00 \_ [ib_cm/5]
2387 ? S< 0:00 \_ [ib_cm/6]
2388 ? S< 0:00 \_ [ib_cm/7]
2398 ? S< 0:00 \_ [rdma_cm]
2684 ? S< 0:00 \_ [bond0]
2882 ? S< 0:00 \_ [bond1]
3195 ? S< 0:00 \_ [kondemand/0]
3197 ? S< 0:00 \_ [kondemand/1]
3198 ? S< 0:00 \_ [kondemand/2]
3199 ? S< 0:00 \_ [kondemand/3]
3200 ? S< 0:00 \_ [kondemand/4]
3201 ? S< 0:00 \_ [kondemand/5]
3202 ? S< 0:00 \_ [kondemand/6]
3203 ? S< 0:00 \_ [kondemand/7]
688 ? S<s 0:00 /sbin/udevd -d
2425 ? S<Lsl 0:00 iscsiuio
2432 ? Ss 0:00 iscsid
2434 ? S<Ls 0:00 iscsid
3061 ? S<sl 0:00 auditd
3063 ? S<sl 0:00 \_ /sbin/audispd
3121 ? Ss 0:00 syslogd -m 0
3124 ? Ss 0:00 klogd -x
3220 ? Ss 0:00 irqbalance
3278 ? Ss 0:00 dbus-daemon --system
3324 ? Ss 0:00 /usr/sbin/acpid
3337 ? Ss 0:00 hald
3338 ? S 0:00 \_ hald-runner
3345 ? S 0:00 \_ hald-addon-acpi: listening on acpid socket /var/run/acpid.socket
3349 ? S 0:00 \_ hald-addon-keyboard: listening on /dev/input/event1
3360 ? S 0:00 \_ hald-addon-storage: polling /dev/sr0
3413 ? Ssl 0:00 automount
3435 ? Ssl 0:00 /usr/sbin/named -u named
3466 ? Ss 0:00 /usr/sbin/sshd
4072 ? Ss 0:00 \_ sshd: root@pts/0
4078 pts/0 Ss 0:00 \_ -bash
5436 pts/0 R+ 0:00 \_ ps axf
3484 ? Ss 0:00 xinetd -stayalive -pidfile /var/run/xinetd.pid
3500 ? SLs 0:00 ntpd -u ntp:ntp -p /var/run/ntpd.pid -g
3514 ? S 0:00 /bin/sh /usr/bin/mysqld_safe --datadir=/var/lib/mysql --pid-file=/var/lib/mysql/server.myserver.com.pid
3575 ? Sl 0:00 \_ /usr/sbin/mysqld --basedir=/ --datadir=/var/lib/mysql --user=mysql --log-error=/var/lib/mysql/server.myserver.com.err --pid-fil
3687 ? Ss 0:00 /usr/sbin/exim -bd -q1h
3709 ? Ss 0:00 /usr/sbin/dovecot
3710 ? S 0:00 \_ dovecot-auth
3725 ? S 0:00 \_ pop3-login
3726 ? S 0:00 \_ pop3-login
3727 ? S 0:00 \_ imap-login
3728 ? S 0:00 \_ imap-login
3729 ? Ss 0:00 /usr/local/apache/bin/httpd -k start -DSSL
4326 ? S 0:00 \_ /usr/bin/perl /usr/local/cpanel/bin/leechprotect
4332 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4333 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4334 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4335 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4336 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4337 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4382 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4383 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
4384 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
5389 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
5390 ? S 0:00 \_ /usr/local/apache/bin/httpd -k start -DSSL
3741 ? Ss 0:00 pure-ftpd (SERVER)
3746 ? S 0:00 /usr/sbin/pure-authd -s /var/run/ftpd.sock -r /usr/sbin/pureauth
3759 ? Ss 0:00 crond
3772 ? Ss 0:00 /usr/sbin/atd
3909 ? S 0:00 cpsrvd (SSL) - waiting for connections
5435 ? Z 0:00 \_ [cpsrvd-ssl] <defunct>
3931 ? S 0:00 queueprocd - wait to process a task
3948 ? S 0:00 tailwatchd
3954 ? SN 0:00 cpanellogd - sleeping for logs
4003 ? Ss 0:00 ./nimbus /opt/nimsoft
4016 ? S 0:00 \_ nimbus(controller)
4053 ? Sl 0:00 \_ nimbus(spooler)
4066 ? S 0:00 \_ nimbus(hdb)
4069 ? S 0:00 \_ nimbus(cdm)
4070 ? S 0:00 \_ nimbus(processes)
4023 ? S 0:00 /usr/sbin/smartd -q never
4027 tty1 Ss+ 0:00 /sbin/mingetty tty1
4028 tty2 Ss+ 0:00 /sbin/mingetty tty2
4029 tty3 Ss+ 0:00 /sbin/mingetty tty3
4030 tty4 Ss+ 0:00 /sbin/mingetty tty4
4031 tty5 Ss+ 0:00 /sbin/mingetty tty5
4033 tty6 Ss+ 0:00 /sbin/mingetty tty6
4035 ttyS1 Ss+ 0:00 /sbin/agetty -h -L ttyS1 19200 vt100
vmstat 10 6
procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------
r b swpd free buff cache si so bi bo in cs us sy id wa st
0 0 0 3718136 25684 257424 0 0 8 3 127 189 0 0 100 0 0
0 0 0 3718136 25700 257420 0 0 0 7 1013 1500 0 0 100 0 0
0 0 0 3718136 25700 257424 0 0 0 1 1013 1551 0 0 100 0 0
0 0 0 3718136 25700 257424 0 0 0 0 1012 1469 0 0 100 0 0
1 0 0 3712680 25716 257424 0 0 0 2 1013 1542 0 0 100 0 0
0 0 0 3718376 25740 257424 0 0 0 46 1017 1534 0 0 100 0 0
Can anyone advise me as to what is the cause of and how I may resolve this behaviour? A kernel/driver conflict perhaps?
I don't see any processes in R or D state that might inflate the load averages artificially, I realise it may be considered low in an 8 thread system but its higher at idle than any normal behaviour I've previously come across.
Thanks in advance for your time.
Edit:
iotop
Total DISK READ: 0.00 B/s | Total DISK WRITE: 0.00 B/s
TID PRIO USER DISK READ DISK WRITE SWAPIN IO> COMMAND
26 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.29 % [events/0]
3205 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.10 % [kondemand/2]
3208 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kondemand/5]
3209 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kondemand/6]
3207 be/3 root 0.00 B/s 0.00 B/s 0.10 % 0.00 % [kondemand/4]
3210 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kondemand/7]
3227 be/4 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % irqbalance
3288 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [rpciod/1]
3287 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [rpciod/0]
3206 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kondemand/3]
3069 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % auditd
3070 be/2 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % audispd
655 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kauditd]
3619 be/4 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % automount
3 be/7 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [ksoftirqd/0]
3068 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % auditd
29 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [events/3]
4 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/0]
7 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/1]
10 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/2]
13 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/3]
16 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/4]
19 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/5]
22 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/6]
25 rt/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [watchdog/7]
27 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [events/1]
28 be/3 root 0.00 B/s 0.00 B/s 0.29 % 0.00 % [events/2]
30 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [events/4]
31 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [events/5]
32 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [events/6]
33 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [events/7]
34 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [khelper]
35 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kthread]
45 be/3 root 0.00 B/s 0.00 B/s 0.00 % 0.00 % [kblockd/0]
