Merge remote-tracking branch 'origin/jetty-9.3.x'

Conflicts:
	jetty-security/src/main/java/org/eclipse/jetty/security/HashLoginService.java

4 files changed, 108 insertions, 60 deletions

diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java
index df6f65f597..1330a1f80b 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java
@@ -787,6 +787,7 @@ public class HttpOutput extends ServletOutputStream implements Runnable
             }
             break;
         }
+        
 
         ByteBuffer buffer = _channel.useDirectBuffers() ? httpContent.getDirectBuffer() : null;
         if (buffer == null)
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceCache.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceCache.java
index a209bf109f..6141c693a1 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceCache.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceCache.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
 import java.nio.channels.ReadableByteChannel;
 import java.util.Comparator;
 import java.util.SortedSet;
@@ -172,7 +173,7 @@ public class ResourceCache implements HttpContent.Factory
     public HttpContent lookup(String pathInContext)
         throws IOException
     {
-        return getContent(pathInContext);
+        return getContent(pathInContext,_maxCachedFileSize);
     }
 
     /* ------------------------------------------------------------ */
@@ -180,6 +181,8 @@ public class ResourceCache implements HttpContent.Factory
      * Get either a valid entry object or create a new one if possible.
      *
      * @param pathInContext The key into the cache
+     * @param maxBuffer The maximum buffer to allocated for this request.  For cached content, a larger buffer may have
+     * previously been allocated and returned by the {@link HttpContent#getDirectBuffer()} or {@link HttpContent#getIndirectBuffer()} calls.
      * @return The entry matching <code>pathInContext</code>, or a new entry 
      * if no matching entry was found. If the content exists but is not cachable, 
      * then a {@link ResourceHttpContent} instance is return. If 
@@ -187,7 +190,7 @@ public class ResourceCache implements HttpContent.Factory
      * @throws IOException Problem loading the resource
      */
     @Override
-    public HttpContent getContent(String pathInContext)
+    public HttpContent getContent(String pathInContext,int maxBufferSize)
         throws IOException
     {
         // Is the content in this cache?
@@ -197,14 +200,14 @@ public class ResourceCache implements HttpContent.Factory
        
         // try loading the content from our factory.
         Resource resource=_factory.getResource(pathInContext);
-        HttpContent loaded = load(pathInContext,resource);
+        HttpContent loaded = load(pathInContext,resource,maxBufferSize);
         if (loaded!=null)
             return loaded;
         
         // Is the content in the parent cache?
         if (_parent!=null)
         {
-            HttpContent httpContent=_parent.lookup(pathInContext);
+            HttpContent httpContent=_parent.getContent(pathInContext,maxBufferSize);
             if (httpContent!=null)
                 return httpContent;
         }
@@ -225,14 +228,13 @@ public class ResourceCache implements HttpContent.Factory
         long len = resource.length();
 
         // Will it fit in the cache?
-        return  (len>0 && len<_maxCachedFileSize && len<_maxCacheSize);
+        return  (len>0 && (_useFileMappedBuffer || (len<_maxCachedFileSize && len<_maxCacheSize)));
     }
     
     /* ------------------------------------------------------------ */
-    private HttpContent load(String pathInContext, Resource resource)
+    private HttpContent load(String pathInContext, Resource resource, int maxBufferSize)
         throws IOException
     {
-        
         if (resource==null || !resource.exists())
             return null;
         
@@ -256,7 +258,6 @@ public class ResourceCache implements HttpContent.Factory
                     if (resourceGz.exists() && resourceGz.lastModified()>=resource.lastModified() && resourceGz.length()<resource.length())
                     {
                         contentGz = new CachedHttpContent(pathInContextGz,resourceGz,null);
-                        shrinkCache();
                         CachedHttpContent added = _cache.putIfAbsent(pathInContextGz,contentGz);
                         if (added!=null)
                         {
@@ -270,9 +271,6 @@ public class ResourceCache implements HttpContent.Factory
             else 
                 content = new CachedHttpContent(pathInContext,resource,null);
 
-            // reduce the cache to an acceptable size.
-            shrinkCache();
-
             // Add it to the cache.
             CachedHttpContent added = _cache.putIfAbsent(pathInContext,content);
             if (added!=null)
@@ -284,7 +282,7 @@ public class ResourceCache implements HttpContent.Factory
             return content;
         }
         
-        // Look for a gzip resource or content
+        // Look for non Cacheable gzip resource or content
         String mt = _mimeTypes.getMimeByExtension(pathInContext);
         if (_gzip)
         {
@@ -292,16 +290,16 @@ public class ResourceCache implements HttpContent.Factory
             String pathInContextGz=pathInContext+".gz";
             CachedHttpContent contentGz = _cache.get(pathInContextGz);
             if (contentGz!=null && contentGz.isValid() && contentGz.getResource().lastModified()>=resource.lastModified())
-                return new ResourceHttpContent(resource,mt,getMaxCachedFileSize(),contentGz);
+                return new ResourceHttpContent(resource,mt,maxBufferSize,contentGz);
             
             // Is there a gzip resource?
             Resource resourceGz=_factory.getResource(pathInContextGz);
             if (resourceGz.exists() && resourceGz.lastModified()>=resource.lastModified() && resourceGz.length()<resource.length())
-                return new ResourceHttpContent(resource,mt,getMaxCachedFileSize(),
-                       new ResourceHttpContent(resourceGz,_mimeTypes.getMimeByExtension(pathInContextGz),getMaxCachedFileSize()));
+                return new ResourceHttpContent(resource,mt,maxBufferSize,
+                       new ResourceHttpContent(resourceGz,_mimeTypes.getMimeByExtension(pathInContextGz),maxBufferSize));
         }
         
-        return new ResourceHttpContent(resource,mt,getMaxCachedFileSize());
+        return new ResourceHttpContent(resource,mt,maxBufferSize);
     }
     
     /* ------------------------------------------------------------ */
@@ -359,6 +357,8 @@ public class ResourceCache implements HttpContent.Factory
     /* ------------------------------------------------------------ */
     protected ByteBuffer getDirectBuffer(Resource resource)
     {
+        // Only use file mapped buffers for cached resources, otherwise too much virtual memory commitment for
+        // a non shared resource.  Also ignore max buffer size
         try
         {
             if (_useFileMappedBuffer && resource.getFile()!=null && resource.length()<Integer.MAX_VALUE) 
@@ -421,8 +421,9 @@ public class ResourceCache implements HttpContent.Factory
             _contentLengthValue=exists?(int)resource.length():0;
             _contentLength=new PreEncodedHttpField(HttpHeader.CONTENT_LENGTH,Long.toString(_contentLengthValue));
             
-            _cachedSize.addAndGet(_contentLengthValue);
-            _cachedFiles.incrementAndGet();
+            if (_cachedFiles.incrementAndGet()>_maxCachedFiles)
+                shrinkCache();
+            
             _lastAccessed=System.currentTimeMillis();
             
             _etag=ResourceCache.this._etags?new PreEncodedHttpField(HttpHeader.ETAG,resource.getWeakETag()):null;
@@ -487,8 +488,14 @@ public class ResourceCache implements HttpContent.Factory
         /* ------------------------------------------------------------ */
         protected void invalidate()
         {
-            // Invalidate it
-            _cachedSize.addAndGet(-_contentLengthValue);
+            ByteBuffer indirect=_indirectBuffer.get();
+            if (indirect!=null && _indirectBuffer.compareAndSet(indirect,null))
+                _cachedSize.addAndGet(-BufferUtil.length(indirect));
+            
+            ByteBuffer direct=_directBuffer.get();
+            if (direct!=null && !BufferUtil.isMappedBuffer(direct) && _directBuffer.compareAndSet(direct,null))
+                _cachedSize.addAndGet(-BufferUtil.length(direct));
+
             _cachedFiles.decrementAndGet();
             _resource.close();
         }
@@ -507,7 +514,6 @@ public class ResourceCache implements HttpContent.Factory
             return _lastModified==null?null:_lastModified.getValue();
         }
 
-
         /* ------------------------------------------------------------ */
         @Override
         public HttpField getContentType()
@@ -569,7 +575,11 @@ public class ResourceCache implements HttpContent.Factory
                 if (buffer2==null)
                     LOG.warn("Could not load "+this);
                 else if (_indirectBuffer.compareAndSet(null,buffer2))
+                {
                     buffer=buffer2;
+                    if (_cachedSize.addAndGet(BufferUtil.length(buffer))>_maxCacheSize)
+                        shrinkCache();
+                }
                 else
                     buffer=_indirectBuffer.get();
             }
@@ -578,7 +588,6 @@ public class ResourceCache implements HttpContent.Factory
             return buffer.slice();
         }
         
-
         /* ------------------------------------------------------------ */
         @Override
         public ByteBuffer getDirectBuffer()
@@ -591,7 +600,12 @@ public class ResourceCache implements HttpContent.Factory
                 if (buffer2==null)
                     LOG.warn("Could not load "+this);
                 else if (_directBuffer.compareAndSet(null,buffer2))
+                {
                     buffer=buffer2;
+
+                    if (!BufferUtil.isMappedBuffer(buffer) && _cachedSize.addAndGet(BufferUtil.length(buffer))>_maxCacheSize)
+                        shrinkCache(); 
+                }
                 else
                     buffer=_directBuffer.get();
             }
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceContentFactory.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceContentFactory.java
index 2e0edde673..11d8810734 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceContentFactory.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceContentFactory.java
@@ -27,55 +27,47 @@ import org.eclipse.jetty.http.ResourceHttpContent;
 import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.resource.ResourceFactory;
 
+
+/**
+ * A HttpContent.Factory for transient content.  The HttpContent's created by 
+ * this factory are not intended to be cached, so memory limits for individual
+ * HttpOutput streams are enforced.
+ */
 public class ResourceContentFactory implements Factory
 {
     private final ResourceFactory _factory;
     private final MimeTypes _mimeTypes;
-    private final int _maxBufferSize;
     private final boolean _gzip;
     
-
     /* ------------------------------------------------------------ */
-    public ResourceContentFactory(ResourceFactory factory, MimeTypes mimeTypes, int maxBufferSize, boolean gzip)
+    public ResourceContentFactory(ResourceFactory factory, MimeTypes mimeTypes, boolean gzip)
     {
         _factory=factory;
         _mimeTypes=mimeTypes;
-        _maxBufferSize=maxBufferSize;
         _gzip=gzip;
     }
 
     /* ------------------------------------------------------------ */
-    /** Get a Entry from the cache.
-     * Get either a valid entry object or create a new one if possible.
-     *
-     * @param pathInContext The key into the cache
-     * @return The entry matching <code>pathInContext</code>, or a new entry 
-     * if no matching entry was found. If the content exists but is not cachable, 
-     * then a {@link ResourceHttpContent} instance is return. If 
-     * the resource does not exist, then null is returned.
-     * @throws IOException Problem loading the resource
-     */
     @Override
-    public HttpContent getContent(String pathInContext)
+    public HttpContent getContent(String pathInContext,int maxBufferSize)
         throws IOException
     {
-       
         // try loading the content from our factory.
         Resource resource=_factory.getResource(pathInContext);
-        HttpContent loaded = load(pathInContext,resource);
+        HttpContent loaded = load(pathInContext,resource,maxBufferSize);
         return loaded;
     }
     
     
     /* ------------------------------------------------------------ */
-    private HttpContent load(String pathInContext, Resource resource)
+    private HttpContent load(String pathInContext, Resource resource, int maxBufferSize)
         throws IOException
     {   
         if (resource==null || !resource.exists())
             return null;
         
         if (resource.isDirectory())
-            return new ResourceHttpContent(resource,_mimeTypes.getMimeByExtension(resource.toString()),_maxBufferSize);
+            return new ResourceHttpContent(resource,_mimeTypes.getMimeByExtension(resource.toString()),maxBufferSize);
         
         // Look for a gzip resource or content
         String mt = _mimeTypes.getMimeByExtension(pathInContext);
@@ -85,11 +77,11 @@ public class ResourceContentFactory implements Factory
             String pathInContextGz=pathInContext+".gz";
             Resource resourceGz=_factory.getResource(pathInContextGz);
             if (resourceGz.exists() && resourceGz.lastModified()>=resource.lastModified() && resourceGz.length()<resource.length())
-                return new ResourceHttpContent(resource,mt,_maxBufferSize,
-                       new ResourceHttpContent(resourceGz,_mimeTypes.getMimeByExtension(pathInContextGz),_maxBufferSize));
+                return new ResourceHttpContent(resource,mt,maxBufferSize,
+                       new ResourceHttpContent(resourceGz,_mimeTypes.getMimeByExtension(pathInContextGz),maxBufferSize));
         }
         
-        return new ResourceHttpContent(resource,mt,_maxBufferSize);
+        return new ResourceHttpContent(resource,mt,maxBufferSize);
     }
     
     
diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java
index 67c6e2447f..fb465b8a94 100644
--- a/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java
+++ b/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java
@@ -20,6 +20,7 @@ package org.eclipse.jetty.server;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -30,6 +31,7 @@ import java.io.OutputStream;
 import org.eclipse.jetty.http.HttpContent;
 import org.eclipse.jetty.http.MimeTypes;
 import org.eclipse.jetty.http.ResourceHttpContent;
+import org.eclipse.jetty.toolchain.test.OS;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.resource.ResourceCollection;
@@ -136,50 +138,83 @@ public class ResourceCacheTest
         cache.setMaxCachedFileSize(85);
         cache.setMaxCachedFiles(4);
 
-        assertTrue(cache.lookup("does not exist")==null);
-        assertTrue(cache.lookup(names[9]) instanceof ResourceHttpContent);
+        assertTrue(cache.getContent("does not exist",4096)==null);
+        assertTrue(cache.getContent(names[9],4096) instanceof ResourceHttpContent);
+        assertTrue(cache.getContent(names[9],4096).getIndirectBuffer()!=null);
 
         HttpContent content;
-        content=cache.lookup(names[8]);
+        content=cache.getContent(names[8],4096);
         assertTrue(content!=null);
         assertEquals(80,content.getContentLengthValue());
-
+        assertEquals(0,cache.getCachedSize());
+        
+        if (OS.IS_LINUX)
+        {
+            // Initially not using memory mapped files
+            content.getDirectBuffer();
+            assertEquals(80,cache.getCachedSize());
+
+            // with both types of buffer loaded, this is too large for cache
+            content.getIndirectBuffer();
+            assertEquals(0,cache.getCachedSize());
+            assertEquals(0,cache.getCachedFiles());
+
+            cache=new ResourceCache(null,directory,new MimeTypes(),true,false,false);
+            cache.setMaxCacheSize(95);
+            cache.setMaxCachedFileSize(85);
+            cache.setMaxCachedFiles(4);
+            
+            content=cache.getContent(names[8],4096);
+            content.getDirectBuffer();
+            assertEquals(cache.isUseFileMappedBuffer()?0:80,cache.getCachedSize());
+
+            // with both types of buffer loaded, this is not too large for cache because
+            // mapped buffers don't count, so we can continue
+        }
+        content.getIndirectBuffer();
         assertEquals(80,cache.getCachedSize());
         assertEquals(1,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[1]);
+        content=cache.getContent(names[1],4096);
+        assertEquals(80,cache.getCachedSize());
+        content.getIndirectBuffer();
         assertEquals(90,cache.getCachedSize());
         assertEquals(2,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[2]);
+        content=cache.getContent(names[2],4096);
+        content.getIndirectBuffer();
         assertEquals(30,cache.getCachedSize());
         assertEquals(2,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[3]);
+        content=cache.getContent(names[3],4096);
+        content.getIndirectBuffer();
         assertEquals(60,cache.getCachedSize());
         assertEquals(3,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[4]);
+        content=cache.getContent(names[4],4096);
+        content.getIndirectBuffer();
         assertEquals(90,cache.getCachedSize());
         assertEquals(3,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[5]);
+        content=cache.getContent(names[5],4096);
+        content.getIndirectBuffer();
         assertEquals(90,cache.getCachedSize());
         assertEquals(2,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[6]);
+        content=cache.getContent(names[6],4096);
+        content.getIndirectBuffer();
         assertEquals(60,cache.getCachedSize());
         assertEquals(1,cache.getCachedFiles());
 
@@ -189,37 +224,43 @@ public class ResourceCacheTest
         {
             out.write(' ');
         }
-        content=cache.lookup(names[7]);
+        content=cache.getContent(names[7],4096);
+        content.getIndirectBuffer();
         assertEquals(70,cache.getCachedSize());
         assertEquals(1,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[6]);
+        content=cache.getContent(names[6],4096);
+        content.getIndirectBuffer();
         assertEquals(71,cache.getCachedSize());
         assertEquals(2,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[0]);
+        content=cache.getContent(names[0],4096);
+        content.getIndirectBuffer();
         assertEquals(72,cache.getCachedSize());
         assertEquals(3,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[1]);
+        content=cache.getContent(names[1],4096);
+        content.getIndirectBuffer();
         assertEquals(82,cache.getCachedSize());
         assertEquals(4,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[2]);
+        content=cache.getContent(names[2],4096);
+        content.getIndirectBuffer();
         assertEquals(32,cache.getCachedSize());
         assertEquals(4,cache.getCachedFiles());
 
         Thread.sleep(200);
 
-        content=cache.lookup(names[3]);
+        content=cache.getContent(names[3],4096);
+        content.getIndirectBuffer();
         assertEquals(61,cache.getCachedSize());
         assertEquals(4,cache.getCachedFiles());