jetty-9 simplified error handling

5 files changed, 222 insertions, 229 deletions

diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannel.java b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannel.java
index d54fb10133..5fba8936dd 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannel.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannel.java
@@ -253,7 +253,7 @@ public class HttpChannel implements HttpParser.RequestHandler, Runnable
                     LOG.warn(String.valueOf(_uri), e);
                     _state.error(e);
                     _request.setHandled(true);
-                    handleError(e);
+                    handleException(e);
                 }
                 finally
                 {
@@ -312,45 +312,6 @@ public class HttpChannel implements HttpParser.RequestHandler, Runnable
         }
     }
 
-    protected boolean complete()
-    {
-        LOG.debug("{} complete", this);
-        if (_state.isCompleting())
-        {
-            _state.completed();
-            if (isExpecting100Continue())
-            {
-                LOG.debug("100-Continue response not sent");
-                // We didn't send 100 continues, but the latest interpretation
-                // of the spec (see httpbis) is that the client will either
-                // send the body anyway, or close.  So we no longer need to
-                // do anything special here other than make the connection not persistent
-                _expect100Continue = false;
-                if (!isCommitted())
-                    _response.addHeader(HttpHeader.CONNECTION.toString(), HttpHeaderValue.CLOSE.toString());
-                else
-                    LOG.warn("Cannot send 'Connection: close' for 100-Continue, response is already committed");
-            }
-
-            if (!_response.isCommitted() && !_request.isHandled())
-                _response.sendError(Response.SC_NOT_FOUND, null, null);
-
-            _request.setHandled(true);
-
-            try
-            {
-                _response.getHttpOutput().close();
-            }
-            catch (IOException x)
-            {
-                // We cannot write the response, so there is no point in calling
-                // response.sendError() since that writes, and we already know we cannot write.
-                LOG.debug("Could not write response", x);
-            }
-        }
-        return _request.getHttpInput().isShutdown();
-    }
-
     /**
      * <p>Sends an error 500, performing a special logic to detect whether the request is suspended,
      * to avoid concurrent writes from the application.</p>
@@ -360,11 +321,11 @@ public class HttpChannel implements HttpParser.RequestHandler, Runnable
      *
      * @param x the Throwable that caused the problem
      */
-    private void handleError(Throwable x)
+    protected void handleException(Throwable x)
     {
-        if (_state.isSuspended())
+        try
         {
-            try
+            if (_state.isSuspended())
             {
                 HttpFields fields = new HttpFields();
                 ResponseInfo info = new ResponseInfo(_request.getHttpVersion(), fields, 0, Response.SC_INTERNAL_SERVER_ERROR, null, _request.isHead());
@@ -372,110 +333,18 @@ public class HttpChannel implements HttpParser.RequestHandler, Runnable
                 if (!committed)
                     LOG.warn("Could not send response error 500, response is already committed");
             }
-            catch (IOException e)
-            {
-                // We tried our best, just log
-                LOG.debug("Could not commit response error 500", e);
-            }
-        }
-        else
-        {
-            _response.sendError(500, null, x.getMessage());
-        }
-    }
-
-    protected void sendError(ResponseInfo info, String extraContent)
-    {
-        int status = info.getStatus();
-        try
-        {
-            String reason = info.getReason();
-            if (reason == null)
-                reason = HttpStatus.getMessage(status);
-
-            // If we are allowed to have a body
-            if (status != Response.SC_NO_CONTENT &&
-                    status != Response.SC_NOT_MODIFIED &&
-                    status != Response.SC_PARTIAL_CONTENT &&
-                    status >= Response.SC_OK)
-            {
-                ErrorHandler errorHandler = null;
-                ContextHandler.Context context = _request.getContext();
-                if (context != null)
-                    errorHandler = context.getContextHandler().getErrorHandler();
-                if (errorHandler == null)
-                    errorHandler = getServer().getBean(ErrorHandler.class);
-                if (errorHandler != null)
-                {
-                    _request.setAttribute(RequestDispatcher.ERROR_STATUS_CODE, new Integer(status));
-                    _request.setAttribute(RequestDispatcher.ERROR_MESSAGE, reason);
-                    _request.setAttribute(RequestDispatcher.ERROR_REQUEST_URI, _request.getRequestURI());
-                    _request.setAttribute(RequestDispatcher.ERROR_SERVLET_NAME, _request.getServletName());
-                    errorHandler.handle(null, _request, _request, _response);
-                }
-                else
-                {
-                    HttpFields fields = info.getHttpFields();
-                    fields.put(HttpHeader.CACHE_CONTROL, "must-revalidate,no-cache,no-store");
-                    fields.put(HttpHeader.CONTENT_TYPE, MimeTypes.Type.TEXT_HTML_8859_1.toString());
-
-                    reason = escape(reason);
-                    String uri = escape(_request.getRequestURI());
-                    extraContent = escape(extraContent);
-
-                    StringBuilder writer = new StringBuilder(2048);
-                    writer.append("<html>\n");
-                    writer.append("<head>\n");
-                    writer.append("<meta http-equiv=\"Content-Type\" content=\"text/html;charset=ISO-8859-1\"/>\n");
-                    writer.append("<title>Error ").append(Integer.toString(status)).append(' ').append(reason).append("</title>\n");
-                    writer.append("</head>\n");
-                    writer.append("<body>\n");
-                    writer.append("<h2>HTTP ERROR: ").append(Integer.toString(status)).append("</h2>\n");
-                    writer.append("<p>Problem accessing ").append(uri).append(". Reason:\n");
-                    writer.append("<pre>").append(reason).append("</pre></p>");
-                    if (extraContent != null)
-                        writer.append("<p>").append(extraContent).append("</p>");
-                    writer.append("<hr /><i><small>Powered by Jetty://</small></i>\n");
-                    writer.append("</body>\n");
-                    writer.append("</html>");
-                    byte[] bytes = writer.toString().getBytes(StringUtil.__ISO_8859_1);
-                    fields.put(HttpHeader.CONTENT_LENGTH, String.valueOf(bytes.length));
-                    _response.getOutputStream().write(bytes);
-                }
-            }
-            else if (status != Response.SC_PARTIAL_CONTENT)
+            else
             {
-                // TODO: not sure why we need to modify the request when writing an error ?
-                // TODO: or modify the response if the error code cannot have a body ?
-                //            _channel.getRequest().getHttpFields().remove(HttpHeader.CONTENT_TYPE);
-                //            _channel.getRequest().getHttpFields().remove(HttpHeader.CONTENT_LENGTH);
-                //            _characterEncoding = null;
-                //            _mimeType = null;
-            }
-
-            complete();
-
-            // TODO: is this needed ?
-            if (_state.isIdle())
-                _state.complete();
-            _request.getHttpInput().shutdown();
+                _request.setAttribute(RequestDispatcher.ERROR_EXCEPTION,x);
+                _request.setAttribute(RequestDispatcher.ERROR_EXCEPTION_TYPE,x.getClass());
+                _response.sendError(500, x.getMessage());
+            } 
         }
-        catch (IOException x)
-        {
-            // We failed to write the error, bail out
-            LOG.debug("Could not write error response " + status, x);
-        }
-    }
-
-    private String escape(String reason)
-    {
-        if (reason != null)
+        catch (IOException e)
         {
-            reason = reason.replaceAll("&", "&amp;");
-            reason = reason.replaceAll("<", "&lt;");
-            reason = reason.replaceAll(">", "&gt;");
+            // We tried our best, just log
+            LOG.debug("Could not commit response error 500", e);
         }
-        return reason;
     }
 
     public boolean isExpecting100Continue()
@@ -634,7 +503,7 @@ public class HttpChannel implements HttpParser.RequestHandler, Runnable
 
                 if (_expect)
                 {
-                    _response.sendError(Response.SC_EXPECTATION_FAILED, null, null);
+                    badMessage(HttpStatus.EXPECTATION_FAILED_417,null);
                     return true;
                 }
 
@@ -653,9 +522,7 @@ public class HttpChannel implements HttpParser.RequestHandler, Runnable
     public boolean content(ByteBuffer ref)
     {
         if (LOG.isDebugEnabled())
-        {
             LOG.debug("{} content {}", this, BufferUtil.toDetailString(ref));
-        }
         _request.getHttpInput().content(ref);
         return true;
     }
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpConnection.java b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpConnection.java
index 8da003da90..4afd57b777 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpConnection.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpConnection.java
@@ -125,46 +125,6 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
         }
     }
 
-    protected boolean readAndParse() throws IOException
-    {
-        // If there is a request buffer, we are re-entering here
-        if (_requestBuffer == null)
-        {
-            _requestBuffer = _bufferPool.acquire(_configuration.getRequestHeaderSize(), false);
-
-            int filled = getEndPoint().fill(_requestBuffer);
-
-            LOG.debug("{} filled {}", this, filled);
-
-            // If we failed to fill
-            if (filled == 0)
-            {
-                // Somebody wanted to read, we didn't so schedule another attempt
-                releaseRequestBuffer();
-                fillInterested();
-                return false;
-            }
-            else if (filled < 0)
-            {
-                _parser.inputShutdown();
-                // We were only filling if fully consumed, so if we have
-                // read -1 then we have nothing to parse and thus nothing that
-                // will generate a response.  If we had a suspended request pending
-                // a response or a request waiting in the buffer, we would not be here.
-                if (getEndPoint().isOutputShutdown())
-                    getEndPoint().close();
-                else
-                    getEndPoint().shutdownOutput();
-                // buffer must be empty and the channel must be idle, so we can release.
-                releaseRequestBuffer();
-                return false;
-            }
-        }
-
-        // Parse the buffer
-        return _parser.parseNext(_requestBuffer);
-    }
-
     /**
      * <p>Parses and handles HTTP messages.</p>
      * <p>This method is called when this {@link Connection} is ready to read bytes from the {@link EndPoint}.
@@ -183,7 +143,43 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
         {
             while (true)
             {
-                if (readAndParse())
+                // If there is a request buffer, we are re-entering here
+                if (BufferUtil.isEmpty(_requestBuffer))
+                {
+                    if (_requestBuffer == null)
+                        _requestBuffer = _bufferPool.acquire(_configuration.getRequestHeaderSize(), false);
+
+                    int filled = getEndPoint().fill(_requestBuffer);
+
+                    LOG.debug("{} filled {}", this, filled);
+
+                    // If we failed to fill
+                    if (filled == 0)
+                    {
+                        // Somebody wanted to read, we didn't so schedule another attempt
+                        releaseRequestBuffer();
+                        fillInterested();
+                        return;
+                    }
+                    else if (filled < 0)
+                    {
+                        _parser.inputShutdown();
+                        // We were only filling if fully consumed, so if we have
+                        // read -1 then we have nothing to parse and thus nothing that
+                        // will generate a response.  If we had a suspended request pending
+                        // a response or a request waiting in the buffer, we would not be here.
+                        if (getEndPoint().isOutputShutdown())
+                            getEndPoint().close();
+                        else
+                            getEndPoint().shutdownOutput();
+                        // buffer must be empty and the channel must be idle, so we can release.
+                        releaseRequestBuffer();
+                        return;
+                    }
+                }
+
+                // Parse the buffer
+                if (_parser.parseNext(_requestBuffer))
                 {
                     // The parser returned true, which indicates the channel is ready to handle a request.
                     // Call the channel and this will either handle the request/response to completion OR,
@@ -250,9 +246,10 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
         // TODO This is always blocking!  One of the important use-cases is to be able to write large static content without a thread
 
         ByteBuffer header = null;
+        ByteBuffer chunk = null;
         out: while (true)
         {
-            HttpGenerator.Result result = _generator.generateResponse(info, header, content, lastContent);
+            HttpGenerator.Result result = _generator.generateResponse(info, header, chunk, content, lastContent);
             if (LOG.isDebugEnabled())
                 LOG.debug("{} generate: {} ({},{},{})@{}",
                         this,
@@ -266,34 +263,40 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
             {
                 case NEED_HEADER:
                 {
-                    if (header != null)
-                        _bufferPool.release(header);
                     header = _bufferPool.acquire(_configuration.getResponseHeaderSize(), false);
                     continue;
                 }
                 case NEED_CHUNK:
                 {
-                    if (header != null)
-                        _bufferPool.release(header);
-                    header = _bufferPool.acquire(HttpGenerator.CHUNK_SIZE, false);
+                    chunk = _bufferPool.acquire(HttpGenerator.CHUNK_SIZE, false);
                     continue;
                 }
                 case FLUSH:
                 {
+                    // Don't write the chunk or the content if this is a HEAD response
                     if (_channel.getRequest().isHead())
                     {
+                        BufferUtil.clear(chunk);
                         BufferUtil.clear(content);
-                        if (BufferUtil.hasContent(header))
-                            blockingWrite(header);
                     }
-                    else if (BufferUtil.hasContent(header))
+                    
+                    // If we have a header
+                    if (BufferUtil.hasContent(header))
                     {
+                        // we know there will not be a chunk, so write either header+content or just the header
                         if (BufferUtil.hasContent(content))
                             blockingWrite(header, content);
                         else
                             blockingWrite(header);
                     }
-                    else  if (BufferUtil.hasContent(content))
+                    else if (BufferUtil.hasContent(chunk))
+                    {
+                        if (BufferUtil.hasContent(content))
+                            blockingWrite(chunk,content);
+                        else
+                            blockingWrite(chunk);
+                    }
+                    else if (BufferUtil.hasContent(content))
                     {
                         blockingWrite(content);
                     }
@@ -306,6 +309,10 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
                 }
                 case DONE:
                 {
+                    if (header!=null)
+                        _bufferPool.release(header);
+                    if (chunk!=null)
+                        _bufferPool.release(chunk);
                     break out;
                 }
                 case CONTINUE:
@@ -433,17 +440,49 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
         @Override
         protected void blockForContent() throws IOException
         {
+            /* We extend the blockForContent method to replace the
+            default implementation of a blocking queue with an implementation
+            that uses the calling thread to block on a readable callback and
+            then to do the parsing before before attempting the read.
+            */
             try
             {
                 while (true)
                 {
-                    FutureCallback<Void> callback = new FutureCallback<>();
-                    getEndPoint().fillInterested(null, callback);
-                    callback.get();
-                    if (readAndParse())
-                        break;
-                    else
-                        releaseRequestBuffer();
+                    // Can the parser progress (even with an empty buffer)
+                    boolean event=_parser.parseNext(_requestBuffer==null?BufferUtil.EMPTY_BUFFER:_requestBuffer);
+
+                    // If there is more content to parse, leep so we can queue all content from this buffer now without the
+                    // need to call blockForContent again
+                    while (BufferUtil.hasContent(_requestBuffer) && _parser.inContentState())
+                        event|=_parser.parseNext(_requestBuffer);
+                    
+                    // If we have an event, return
+                    if (event)
+                        return;
+                    
+                    // Do we have content ready to parse?
+                    if (BufferUtil.isEmpty(_requestBuffer))
+                    {
+                        // Wait until we can read
+                        FutureCallback<Void> block=new FutureCallback<>();
+                        getEndPoint().fillInterested(null,block);
+                        LOG.debug("{} block readable on {}",this,block);
+                        block.get();
+
+                        // We will need a buffer to read into
+                        if (_requestBuffer==null)
+                            _requestBuffer=_bufferPool.acquire(_configuration.getRequestBufferSize(),false);
+
+                        // read some data
+                        int filled=getEndPoint().fill(_requestBuffer);
+                        LOG.debug("{} block filled {}",this,filled);
+                        if (filled<0)
+                        {
+                            _parser.inputShutdown();
+                            return;
+                        }
+                    }  
                 }
             }
             catch (InterruptedException x)
@@ -510,6 +549,13 @@ public class HttpConnection extends AbstractConnection implements Runnable, Http
                 _generator.setPersistent(false);
             return result;
         }
+
+        @Override
+        protected void handleException(Throwable x)
+        {
+            _generator.setPersistent(false);
+            super.handleException(x);
+        }
         
     }
     
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/LocalConnector.java b/jetty-server/src/main/java/org/eclipse/jetty/server/LocalConnector.java
index 208a0487d8..be58581a78 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/LocalConnector.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/LocalConnector.java
@@ -238,7 +238,10 @@ public class LocalConnector extends AbstractConnector
                     if (!_closed.await(idleFor,units))
                     {
                         if (size==getOutput().remaining())
+                        {
+                            LOG.debug("idle for {} {}",idleFor,units);
                             return;
+                        }
                         size=getOutput().remaining();
                     }
                 }
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/Response.java b/jetty-server/src/main/java/org/eclipse/jetty/server/Response.java
index 9ffe60307d..f925d7c295 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/Response.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/Response.java
@@ -25,6 +25,8 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.servlet.RequestDispatcher;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletResponse;
@@ -41,6 +43,9 @@ import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MimeTypes;
+import org.eclipse.jetty.server.handler.ContextHandler;
+import org.eclipse.jetty.server.handler.ErrorHandler;
+import org.eclipse.jetty.util.ByteArrayISO8859Writer;
 import org.eclipse.jetty.util.StringUtil;
 import org.eclipse.jetty.util.URIUtil;
 import org.eclipse.jetty.util.log.Log;
@@ -298,35 +303,101 @@ public class Response implements HttpServletResponse
     @Override
     public void sendError(int code, String message) throws IOException
     {
-        sendError(code, message, null);
-    }
-
-    protected void sendError(int code, String message, String content)
-    {
         if (isIncluding())
             return;
 
         if (isCommitted())
-        {
-            LOG.warn("Could not commit error {} {}, response already committed", code, message);
-            return;
-        }
+            LOG.warn("Committed before "+code+" "+message);
 
         resetBuffer();
-        _characterEncoding = null;
-        setHeader(HttpHeader.EXPIRES, null);
-        setHeader(HttpHeader.LAST_MODIFIED, null);
-        setHeader(HttpHeader.CACHE_CONTROL, null);
-        setHeader(HttpHeader.CONTENT_TYPE, null);
-        setHeader(HttpHeader.CONTENT_LENGTH, null);
-
-        if (message == null)
-            message = HttpStatus.getMessage(code);
-        setStatus(code, message);
+        _characterEncoding=null;
+        setHeader(HttpHeader.EXPIRES,null);
+        setHeader(HttpHeader.LAST_MODIFIED,null);
+        setHeader(HttpHeader.CACHE_CONTROL,null);
+        setHeader(HttpHeader.CONTENT_TYPE,null);
+        setHeader(HttpHeader.CONTENT_LENGTH,null);
 
         _outputType = OutputType.NONE;
+        setStatus(code);
+        _reason=message;
+
+        if (message==null)
+            message=HttpStatus.getMessage(code);
+
+        // If we are allowed to have a body
+        if (code!=SC_NO_CONTENT &&
+            code!=SC_NOT_MODIFIED &&
+            code!=SC_PARTIAL_CONTENT &&
+            code>=SC_OK)
+        {
+            Request request = _channel.getRequest();
+
+            ErrorHandler error_handler = null;
+            ContextHandler.Context context = request.getContext();
+            if (context!=null)
+                error_handler=context.getContextHandler().getErrorHandler();
+            if (error_handler==null)
+                error_handler = _channel.getServer().getBean(ErrorHandler.class);
+            if (error_handler!=null)
+            {
+                request.setAttribute(RequestDispatcher.ERROR_STATUS_CODE,new Integer(code));
+                request.setAttribute(RequestDispatcher.ERROR_MESSAGE, message);
+                request.setAttribute(RequestDispatcher.ERROR_REQUEST_URI, request.getRequestURI());
+                request.setAttribute(RequestDispatcher.ERROR_SERVLET_NAME,request.getServletName());
+                error_handler.handle(null,_channel.getRequest(),_channel.getRequest(),this );
+            }
+            else
+            {
+                setHeader(HttpHeader.CACHE_CONTROL, "must-revalidate,no-cache,no-store");
+                setContentType(MimeTypes.Type.TEXT_HTML_8859_1.toString());
+                ByteArrayISO8859Writer writer= new ByteArrayISO8859Writer(2048);
+                if (message != null)
+                {
+                    message= StringUtil.replace(message, "&", "&amp;");
+                    message= StringUtil.replace(message, "<", "&lt;");
+                    message= StringUtil.replace(message, ">", "&gt;");
+                }
+                String uri= request.getRequestURI();
+                if (uri!=null)
+                {
+                    uri= StringUtil.replace(uri, "&", "&amp;");
+                    uri= StringUtil.replace(uri, "<", "&lt;");
+                    uri= StringUtil.replace(uri, ">", "&gt;");
+                }
 
-        _channel.sendError(newResponseInfo(), content);
+                writer.write("<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html;charset=ISO-8859-1\"/>\n");
+                writer.write("<title>Error ");
+                writer.write(Integer.toString(code));
+                writer.write(' ');
+                if (message==null)
+                    message=HttpStatus.getMessage(code);
+                writer.write(message);
+                writer.write("</title>\n</head>\n<body>\n<h2>HTTP ERROR: ");
+                writer.write(Integer.toString(code));
+                writer.write("</h2>\n<p>Problem accessing ");
+                writer.write(uri);
+                writer.write(". Reason:\n<pre>    ");
+                writer.write(message);
+                writer.write("</pre>");
+                writer.write("</p>\n<hr /><i><small>Powered by Jetty://</small></i>");
+                writer.write("\n</body>\n</html>\n");
+
+                writer.flush();
+                setContentLength(writer.size());
+                writer.writeTo(getOutputStream());
+                writer.destroy();
+            }
+        }
+        else if (code!=SC_PARTIAL_CONTENT)
+        {
+            // TODO work out why this is required?
+            _channel.getRequest().getHttpFields().remove(HttpHeader.CONTENT_TYPE);
+            _channel.getRequest().getHttpFields().remove(HttpHeader.CONTENT_LENGTH);
+            _characterEncoding=null;
+            _mimeType=null;
+        }
+
+        complete();
     }
 
     /**
@@ -530,7 +601,10 @@ public class Response implements HttpServletResponse
     @Override
     public void setStatus(int sc)
     {
-        setStatus(sc, null);
+        if (sc <= 0)
+            throw new IllegalArgumentException();
+        if (!isIncluding())
+            _status = sc;
     }
 
     @Override
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/Server.java b/jetty-server/src/main/java/org/eclipse/jetty/server/Server.java
index 28d541f18a..c9c4e0d2d1 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/Server.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/Server.java
@@ -351,10 +351,13 @@ public class Server extends HandlerWrapper implements Attributes
             
             for (Connector connector : _connectors)
             {
-                while (connector.getStatistics().isRunning() && connector.getStatistics().getConnectionsOpen()>0 && System.currentTimeMillis()<stop_by)
-                {
-                    System.err.println(((Dumpable)connector).dump());
-                    Thread.sleep(100);
+                if (connector.getStatistics().isRunning() && connector.getStatistics().getConnectionsOpen()>0 && System.currentTimeMillis()<stop_by)
+                {                    
+                    if (LOG.isDebugEnabled())
+                        System.err.println(((Dumpable)connector).dump());
+                    LOG.info("Waiting for connections to close on {}...",connector);
+                    while (connector.getStatistics().isRunning() && connector.getStatistics().getConnectionsOpen()>0 && System.currentTimeMillis()<stop_by)
+                        Thread.sleep(100);
                 }
             }
         }