We're experiencing a strange problem with our current Varnish configuration.

4x Web Servers (IIS 6.5 on Windows 2003 Server, each installed on a Intel(R) Xeon(R) CPU E5450 @ 3.00GHz Quad Core, 4GB RAM)

3x Varnish Servers (varnish-3.0.3 revision 9e6a70f on Ubuntu 12.04.2 LTS - 64 bit/precise, Kernel Linux 3.2.0-29-generic, each installed on a Intel(R) Xeon(R) CPU E5450 @ 3.00GHz Quad Core, 4GB RAM)

The Varnish Servers performance are awfully bad in general, to the point that if we shut down one of them the other two are unable to fullfill all the requests and start to skip beats resulting in pending requests, timeouts, 404, etc.

What can we do to improve our Varnish performance? Considering that we're getting less than 5k request per seconds during our max peak, we should be able to serve our pages even with a single one of them without any problem.

We use a standard, vanilla CFG, as shown by this varnishadm >param.show output:

acceptor_sleep_decay        0.900000 []
acceptor_sleep_incr         0.001000 [s]
acceptor_sleep_max          0.050000 [s]
auto_restart                on [bool]
ban_dups                    on [bool]
ban_lurker_sleep            0.010000 [s]
between_bytes_timeout       60.000000 [s]
cc_command                  "exec gcc -std=gnu99 -g -O2 -pthread -fpic -shared -                                                                                                                               Wl,-x -o %o %s"
cli_buffer                  8192 [bytes]
cli_timeout                 20 [seconds]
clock_skew                  10 [s]
connect_timeout             0.700000 [s]
critbit_cooloff             180.000000 [s]
default_grace               10.000000 [seconds]
default_keep                0.000000 [seconds]
default_ttl                 120.000000 [seconds]
diag_bitmap                 0x0 [bitmap]
esi_syntax                  0 [bitmap]
expiry_sleep                1.000000 [seconds]
fetch_chunksize             128 [kilobytes]
fetch_maxchunksize          262144 [kilobytes]
first_byte_timeout          60.000000 [s]
group                       varnish (113)
gzip_level                  6 []
gzip_memlevel               8 []
gzip_stack_buffer           32768 [Bytes]
gzip_tmp_space              0 []
gzip_window                 15 []
http_gzip_support           off [bool]
http_max_hdr                64 [header lines]
http_range_support          on [bool]
http_req_hdr_len            8192 [bytes]
http_req_size               32768 [bytes]
http_resp_hdr_len           8192 [bytes]
http_resp_size              32768 [bytes]
idle_send_timeout           60 [seconds]
listen_address              :80
listen_depth                1024 [connections]
log_hashstring              on [bool]
log_local_address           off [bool]
lru_interval                2 [seconds]
max_esi_depth               5 [levels]
max_restarts                4 [restarts]
nuke_limit                  50 [allocations]
pcre_match_limit            10000 []
pcre_match_limit_recursion  10000 []
ping_interval               3 [seconds]
pipe_timeout                60 [seconds]
prefer_ipv6                 off [bool]
queue_max                   100 [%]
rush_exponent               3 [requests per request]
saintmode_threshold         10 [objects]
send_timeout                600 [seconds]
sess_timeout                5 [seconds]
sess_workspace              16384 [bytes]
session_linger              50 [ms]
session_max                 100000 [sessions]
shm_reclen                  255 [bytes]
shm_workspace               8192 [bytes]
shortlived                  10.000000 [s]
syslog_cli_traffic          on [bool]
thread_pool_add_delay       2 [milliseconds]
thread_pool_add_threshold   2 [requests]
thread_pool_fail_delay      200 [milliseconds]
thread_pool_max             2000 [threads]
thread_pool_min             5 [threads]
thread_pool_purge_delay     1000 [milliseconds]
thread_pool_stack           unlimited [bytes]
thread_pool_timeout         300 [seconds]
thread_pool_workspace       65536 [bytes]
thread_pools                2 [pools]
thread_stats_rate           10 [requests]
user                        varnish (106)
vcc_err_unref               on [bool]
vcl_dir                     /etc/varnish
vcl_trace                   off [bool]
vmod_dir                    /usr/lib/varnish/vmods
waiter                      default (epoll, poll)

This is our default.vcl file: LINK

sub vcl_recv {

        # BASIC recv COMMANDS:
        #
        # lookup -> search the item in the cache
        # pass -> always serve a fresh item (no-caching)
        # pipe -> like pass but ensures a direct-connection with the backend (no-cache AND no-proxy)

        # Allow the backend to serve up stale content if it is responding slow.
        # This defines when Varnish should use a stale object if it has one in the cache.
        set req.grace = 30s;

        if (client.ip == "127.0.0.1") {
                # request from NGINX - do not alter X-Forwarded-For
                set req.http.HTTPS = "on";
        }
        else {
                # Add an X-Forwarded-For to keep track of original request
                unset req.http.HTTPS;
                unset req.http.X-Forwarded-For;
                set req.http.X-Forwarded-For = client.ip;
        }
        set req.backend = www_director;

        # Strip all cookies to force an anonymous request when the back-end servers are down.

        if (!req.backend.healthy) {
          unset req.http.Cookie;
        }

        ## HHTP Accept-Encoding
        if (req.http.Accept-Encoding) {
                if (req.http.Accept-Encoding ~ "gzip") {
                        set req.http.Accept-Encoding = "gzip";
                }
                else if (req.http.Accept-Encoding ~ "deflate") {
                        set req.http.Accept-Encoding = "deflate";
                }
                else {
                        unset req.http.Accept-Encoding;
                }
        }

        if (req.request != "GET" &&
            req.request != "HEAD" &&
            req.request != "PUT" &&
            req.request != "POST" &&
            req.request != "TRACE" &&
            req.request != "OPTIONS" &&
            req.request != "DELETE") {
            /* non-RFC2616 or CONNECT */
               return (pipe);
        }

        if (req.request != "GET" && req.request != "HEAD") {
          /* only deal with GET and HEAD by default */
          return (pass);
        }

        if (req.http.Authorization) {
          return (pass);
        }

        if (req.http.HTTPS ~ "on") {
          return (pass);
        }


######################################################
# COOKIE HANDLING
######################################################

# METHOD 1: do not remove cookies, but pass the page if they contain TB_NC

        if (!(req.url ~ "(?i)\.(png|gif|ipeg|jpg|ico|swf|css|js)(\?[a-z0-9]+)?$")) {
                if (req.http.Cookie && req.http.Cookie ~ "TB_NC") {
                        return (pass);
                }
        }
        return (lookup);
}


# Code determining what to do when serving items from the IIS Server

sub vcl_fetch {
        unset beresp.http.Server;
        set beresp.http.Server = "Server-1";
  # Allow items to be stale if needed. This is the maximum time Varnish should keep an object.
  set beresp.grace = 1h;

  if (req.url ~ "(?i)\.(png|gif|ipeg|jpg|ico|swf|css|js)(\?[a-z0-9]+)?$") {
     unset beresp.http.set-cookie;
  }

  # Default Varnish VCL logic
  if (!beresp.cacheable ||
        beresp.ttl <= 0s ||
        beresp.http.Set-Cookie ||
        beresp.http.Vary == "*") {
      set beresp.ttl = 120 s;
      return(hit_for_pass);
  }

  # Not Cacheable if it has specific TB_NC no-caching cookie
  if (req.http.Cookie && req.http.Cookie ~ "TB_NC") {
    set beresp.http.X-Cacheable = "NO:Got Cookie";
    set beresp.ttl = 120 s;
    return(hit_for_pass);
  }

  # Not Cacheable if it has Cache-Control private
  else if (beresp.http.Cache-Control ~ "private") {
    set beresp.http.X-Cacheable = "NO:Cache-Control=private";
    set beresp.ttl = 120 s;
    return(hit_for_pass);
  }

  # Not Cacheable if it has Cache-Control no-cache or Pragma no-cache
  else if (beresp.http.Cache-Control ~ "no-cache" || beresp.http.Pragma ~ "no-cache") {
    set beresp.http.X-Cacheable = "NO:Cache-Control=no-cache (or pragma no-cache)";
    set beresp.ttl = 120 s;
    return(hit_for_pass);
  }

  # If we reach to this point, the object is cacheable.

  # Cacheable but with not enough ttl: we need to extend the lifetime of the object artificially
  # NOTE: Varnish default TTL is set in /etc/sysconfig/varnish
  #       and can be checked using the following command:
  #       varnishadm param.show default_ttl

  else if (beresp.ttl < 1s) {
    set beresp.ttl   = 5s;
    set beresp.grace = 5s;
    set beresp.http.X-Cacheable = "YES:FORCED";
  }

  # Cacheable and with valid TTL.
  else {
    set beresp.http.X-Cacheable = "YES";
  }

# DEBUG INFO (Cookies)
# set beresp.http.X-Cookie-Debug = "Request cookie: " + req.http.Cookie;
  return(deliver);
}

sub vcl_error {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    if (obj.status == 404) {
        synthetic {"
            <!-- Markup for the 404 page goes here -->
        "};
    }
    else if (obj.status == 500) {
        synthetic {"
            <!-- Markup for the 500 page goes here -->
        "};
    }
    else if (obj.status == 503) {
      if (req.restarts < 4) { return(restart); }
      else {
         synthetic {"
             <!-- Markup for the 503 page goes here -->
         "};
      }
    }
    else {
        synthetic {"
            <!-- Markup for a generic error page goes here -->
        "};
    }
}

sub vcl_deliver {
    if (obj.hits > 0) {
      set resp.http.X-Cache = "HIT";
    } else {
      set resp.http.X-Cache = "MISS";
    }
}

Thanks in advance,