I've got a server with LVM over RAID1. One of the volumes has a consistently higher IO latency (as measured by the diskstats_latency munin plugin) than the other volumes from the same group.

As you can see, the dark orange /root volume has consistently high IO latency. Actually ten times the average latency of the physical devices. It also has the highest Min and Max values. My main concern are not the peaks, which occur under high load, but the constant load on (semi-)idle.

The server is running Debian Squeeze with the VServer kernel and has four VServer containers and one KVM guest.

I'm looking for ways to fix - or at least understand - this situation.

Here're some parts of the system configuration:

root@kvmhost2:~# df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/system--host-root
                       19G  3.8G   14G  22% /
tmpfs                  16G     0   16G   0% /lib/init/rw
udev                   16G  224K   16G   1% /dev
tmpfs                  16G     0   16G   0% /dev/shm
/dev/md0              942M   37M  858M   5% /boot
/dev/mapper/system--host-isos
                       28G   19G  8.1G  70% /srv/isos
/dev/mapper/system--host-vs_a
                       30G   23G  6.0G  79% /var/lib/vservers/a
/dev/mapper/system--host-vs_b
                      5.0G  594M  4.1G  13% /var/lib/vservers/b
/dev/mapper/system--host-vs_c
                      5.0G  555M  4.2G  12% /var/lib/vservers/c
/dev/loop0            4.4G  4.4G     0 100% /media/debian-6.0.0-amd64-DVD-1
/dev/loop1            4.4G  4.4G     0 100% /media/debian-6.0.0-i386-DVD-1
/dev/mapper/system--host-vs_d
                       74G   55G   16G  78% /var/lib/vservers/d
root@kvmhost2:~# cat /proc/mounts
rootfs / rootfs rw 0 0
none /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
none /proc proc rw,nosuid,nodev,noexec,relatime 0 0
none /dev devtmpfs rw,relatime,size=16500836k,nr_inodes=4125209,mode=755 0 0
none /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
/dev/mapper/system--host-root / ext3 rw,relatime,errors=remount-ro,data=ordered 0 0
tmpfs /lib/init/rw tmpfs rw,nosuid,relatime,mode=755 0 0
tmpfs /dev/shm tmpfs rw,nosuid,nodev,relatime 0 0
fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0
/dev/md0 /boot ext3 rw,sync,relatime,errors=continue,data=ordered 0 0
/dev/mapper/system--host-isos /srv/isos ext3 rw,relatime,errors=continue,data=ordered 0 0
/dev/mapper/system--host-vs_a /var/lib/vservers/a ext3 rw,relatime,errors=continue,data=ordered 0 0
/dev/mapper/system--host-vs_b /var/lib/vservers/b ext3 rw,relatime,errors=continue,data=ordered 0 0
/dev/mapper/system--host-vs_c /var/lib/vservers/c ext3 rw,relatime,errors=continue,data=ordered 0 0
/dev/loop0 /media/debian-6.0.0-amd64-DVD-1 iso9660 ro,relatime 0 0
/dev/loop1 /media/debian-6.0.0-i386-DVD-1 iso9660 ro,relatime 0 0
/dev/mapper/system--host-vs_d /var/lib/vservers/d ext3 rw,relatime,errors=continue,data=ordered 0 0
root@kvmhost2:~# cat /proc/mdstat
Personalities : [raid1]
md1 : active raid1 sda2[0] sdb2[1]
      975779968 blocks [2/2] [UU]

md0 : active raid1 sda1[0] sdb1[1]
      979840 blocks [2/2] [UU]

unused devices: <none>
root@kvmhost2:~# iostat -x
Linux 2.6.32-5-vserver-amd64 (kvmhost2)         06/28/2012      _x86_64_        (8 CPU)

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           3.09    0.14    2.92    1.51    0.00   92.35

Device:         rrqm/s   wrqm/s     r/s     w/s   rsec/s   wsec/s avgrq-sz avgqu-sz   await  svctm  %util
sda              23.25   161.12    7.46   37.90   855.27  1596.62    54.05     0.13    2.80   1.76   8.00
sdb              22.82   161.36    7.36   37.66   850.29  1596.62    54.35     0.54   12.01   1.80   8.09
md0               0.00     0.00    0.00    0.00     0.14     0.02    38.44     0.00    0.00   0.00   0.00
md1               0.00     0.00   53.55  198.16   768.01  1585.25     9.35     0.00    0.00   0.00   0.00
dm-0              0.00     0.00    0.48   20.21    16.70   161.71     8.62     0.26   12.72   0.77   1.60
dm-1              0.00     0.00    3.62   10.03    28.94    80.21     8.00     0.19   13.68   1.59   2.17
dm-2              0.00     0.00    0.00    0.00     0.00     0.00     9.17     0.00    9.64   6.42   0.00
dm-3              0.00     0.00    6.73    0.41    53.87     3.28     8.00     0.02    3.44   0.12   0.09
dm-4              0.00     0.00   17.45   18.18   139.57   145.47     8.00     0.42   11.81   0.76   2.69
dm-5              0.00     0.00    2.50   46.38   120.50   371.07    10.06     0.69   14.20   0.46   2.26
dm-6              0.00     0.00    0.02    0.10     0.67     0.81    12.53     0.01   75.53  18.58   0.22
dm-7              0.00     0.00    0.00    0.00     0.00     0.00     7.99     0.00   11.24   9.45   0.00
dm-8              0.00     0.00   22.69  102.76   407.25   822.09     9.80     0.97    7.71   0.39   4.95
dm-9              0.00     0.00    0.06    0.08     0.50     0.62     8.00     0.07  481.23  11.72   0.16

root@kvmhost2:~# ls -l /dev/mapper/
total 0
crw------- 1 root root 10, 59 May 11 11:19 control
lrwxrwxrwx 1 root root      7 Jun  5 15:08 system--host-kvm1 -> ../dm-4
lrwxrwxrwx 1 root root      7 Jun  5 15:08 system--host-kvm2 -> ../dm-3
lrwxrwxrwx 1 root root      7 Jun  5 15:06 system--host-isos -> ../dm-2
lrwxrwxrwx 1 root root      7 May 11 11:19 system--host-root -> ../dm-0
lrwxrwxrwx 1 root root      7 Jun  5 15:06 system--host-swap -> ../dm-9
lrwxrwxrwx 1 root root      7 Jun  5 15:06 system--host-vs_d -> ../dm-8
lrwxrwxrwx 1 root root      7 Jun  5 15:06 system--host-vs_b -> ../dm-6
lrwxrwxrwx 1 root root      7 Jun  5 15:06 system--host-vs_c -> ../dm-7
lrwxrwxrwx 1 root root      7 Jun  5 15:06 system--host-vs_a -> ../dm-5
lrwxrwxrwx 1 root root      7 Jun  5 15:08 system--host-kvm3 -> ../dm-1
root@kvmhost2:~#