[201265] Handling large collections by loading CDOIDs in chunks

https://bugs.eclipse.org/bugs/show_bug.cgi?id=201265

4 files changed, 283 insertions, 3 deletions

diff --git a/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/RevisionManager.java b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/RevisionManager.java
index a26811ac47..5077568e1f 100644
--- a/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/RevisionManager.java
+++ b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/RevisionManager.java
@@ -10,15 +10,21 @@
  **************************************************************************/
 package org.eclipse.emf.cdo.internal.server;
 
+import org.eclipse.emf.cdo.internal.protocol.model.CDOClassImpl;
 import org.eclipse.emf.cdo.internal.protocol.model.CDOClassRefImpl;
+import org.eclipse.emf.cdo.internal.protocol.model.CDOFeatureImpl;
 import org.eclipse.emf.cdo.internal.protocol.model.resource.CDOPathFeatureImpl;
 import org.eclipse.emf.cdo.internal.protocol.revision.CDORevisionImpl;
 import org.eclipse.emf.cdo.internal.protocol.revision.CDORevisionResolverImpl;
+import org.eclipse.emf.cdo.internal.protocol.revision.CDORevisionImpl.MoveableList;
 import org.eclipse.emf.cdo.protocol.CDOID;
+import org.eclipse.emf.cdo.protocol.model.CDOFeature;
 import org.eclipse.emf.cdo.protocol.revision.CDOReferenceProxy;
 import org.eclipse.emf.cdo.server.IRevisionManager;
+import org.eclipse.emf.cdo.server.IStoreChunkReader;
 import org.eclipse.emf.cdo.server.IStoreReader;
 import org.eclipse.emf.cdo.server.IStoreWriter;
+import org.eclipse.emf.cdo.server.IStoreChunkReader.Chunk;
 
 import org.eclipse.net4j.util.transaction.ITransaction;
 import org.eclipse.net4j.util.transaction.ITransactionalOperation;
@@ -64,18 +70,120 @@ public class RevisionManager extends CDORevisionResolverImpl implements IRevisio
   }
 
   @Override
-  protected CDORevisionImpl verifyRevision(CDORevisionImpl revision)
+  protected CDORevisionImpl verifyRevision(CDORevisionImpl revision, int referenceChunk)
   {
-    revision = super.verifyRevision(revision);
+    IStoreReader storeReader = null;
+    revision = super.verifyRevision(revision, referenceChunk);
     if (repository.isVerifyingRevisions())
     {
-      IStoreReader storeReader = StoreUtil.getReader();
+      storeReader = StoreUtil.getReader();
       revision = storeReader.verifyRevision(revision);
     }
 
+    ensureChunks(revision, referenceChunk, storeReader);
     return revision;
   }
 
+  protected void ensureChunks(CDORevisionImpl revision, int referenceChunk, IStoreReader storeReader)
+  {
+    CDOClassImpl cdoClass = revision.getCDOClass();
+    CDOFeatureImpl[] features = cdoClass.getAllFeatures();
+    for (int i = 0; i < features.length; i++)
+    {
+      CDOFeatureImpl feature = features[i];
+      if (feature.isReference() && feature.isMany())
+      {
+        MoveableList list = revision.getList(feature);
+        int chunkEnd = Math.min(referenceChunk, list.size());
+        storeReader = ensureChunk(revision, feature, storeReader, list, 0, chunkEnd);
+      }
+    }
+  }
+
+  protected IStoreReader ensureChunk(CDORevisionImpl revision, CDOFeature feature, IStoreReader storeReader,
+      MoveableList list, int chunkStart, int chunkEnd)
+  {
+    IStoreChunkReader chunkReader = null;
+    int fromIndex = -1;
+    for (int j = chunkStart; j < chunkEnd; j++)
+    {
+      if (list.get(j) == CDORevisionImpl.UNINITIALIZED)
+      {
+        if (fromIndex == -1)
+        {
+          fromIndex = j;
+        }
+      }
+      else
+      {
+        if (fromIndex != -1)
+        {
+          if (chunkReader == null)
+          {
+            if (storeReader == null)
+            {
+              storeReader = StoreUtil.getReader();
+            }
+
+            chunkReader = storeReader.createChunkReader(revision, feature);
+          }
+
+          int toIndex = j - 1;
+          if (fromIndex == toIndex)
+          {
+            chunkReader.addSimpleChunk(fromIndex);
+          }
+          else
+          {
+            chunkReader.addRangedChunk(fromIndex, toIndex);
+          }
+
+          fromIndex = -1;
+        }
+      }
+    }
+
+    // Add last chunk
+    if (fromIndex != -1)
+    {
+      if (chunkReader == null)
+      {
+        if (storeReader == null)
+        {
+          storeReader = StoreUtil.getReader();
+        }
+
+        chunkReader = storeReader.createChunkReader(revision, feature);
+      }
+
+      int toIndex = chunkEnd - 1;
+      if (fromIndex == toIndex)
+      {
+        chunkReader.addSimpleChunk(fromIndex);
+      }
+      else
+      {
+        chunkReader.addRangedChunk(fromIndex, toIndex);
+      }
+    }
+
+    if (chunkReader != null)
+    {
+      List<Chunk> chunks = chunkReader.executeRead();
+      for (Chunk chunk : chunks)
+      {
+        int startIndex = chunk.getStartIndex();
+        for (int indexInChunk = 0; indexInChunk < chunk.size(); indexInChunk++)
+        {
+          CDOID id = chunk.getID(indexInChunk);
+          list.set(startIndex + indexInChunk, id);
+        }
+      }
+    }
+
+    return storeReader;
+  }
+
   @Override
   protected CDORevisionImpl loadRevision(CDOID id, int referenceChunk)
   {
diff --git a/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/StoreChunkReader.java b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/StoreChunkReader.java
new file mode 100644
index 0000000000..44aed4c3bc
--- /dev/null
+++ b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/internal/server/StoreChunkReader.java
@@ -0,0 +1,70 @@
+/***************************************************************************
+ * Copyright (c) 2004 - 2007 Eike Stepper, Germany.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors:
+ *    Eike Stepper - initial API and implementation
+ **************************************************************************/
+package org.eclipse.emf.cdo.internal.server;
+
+import org.eclipse.emf.cdo.protocol.model.CDOFeature;
+import org.eclipse.emf.cdo.protocol.revision.CDORevision;
+import org.eclipse.emf.cdo.server.IStoreAccessor;
+import org.eclipse.emf.cdo.server.IStoreChunkReader;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Eike Stepper
+ */
+public abstract class StoreChunkReader implements IStoreChunkReader
+{
+  private IStoreAccessor storeAccessor;
+
+  private CDORevision revision;
+
+  private CDOFeature feature;
+
+  private List<Chunk> chunks = new ArrayList<Chunk>(0);
+
+  public StoreChunkReader(IStoreAccessor storeAccessor, CDORevision revision, CDOFeature feature)
+  {
+    this.storeAccessor = storeAccessor;
+    this.revision = revision;
+    this.feature = feature;
+  }
+
+  public IStoreAccessor getStoreAccessor()
+  {
+    return storeAccessor;
+  }
+
+  public CDORevision getRevision()
+  {
+    return revision;
+  }
+
+  public CDOFeature getFeature()
+  {
+    return feature;
+  }
+
+  public List<Chunk> getChunks()
+  {
+    return chunks;
+  }
+
+  public void addSimpleChunk(int index)
+  {
+    chunks.add(new Chunk(index));
+  }
+
+  public void addRangedChunk(int fromIndex, int toIndex)
+  {
+    chunks.add(new Chunk(fromIndex, toIndex - fromIndex));
+  }
+}
diff --git a/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreChunkReader.java b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreChunkReader.java
new file mode 100644
index 0000000000..96c7ff8c1d
--- /dev/null
+++ b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreChunkReader.java
@@ -0,0 +1,99 @@
+/***************************************************************************
+ * Copyright (c) 2004 - 2007 Eike Stepper, Germany.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors:
+ *    Eike Stepper - initial API and implementation
+ **************************************************************************/
+package org.eclipse.emf.cdo.server;
+
+import org.eclipse.emf.cdo.protocol.CDOID;
+import org.eclipse.emf.cdo.protocol.model.CDOFeature;
+import org.eclipse.emf.cdo.protocol.revision.CDORevision;
+
+import java.util.List;
+
+/**
+ * @author Eike Stepper
+ */
+public interface IStoreChunkReader
+{
+  public CDORevision getRevision();
+
+  public CDOFeature getFeature();
+
+  public void addSimpleChunk(int index);
+
+  public void addRangedChunk(int fromIndex, int toIndex);
+
+  public List<Chunk> executeRead();
+
+  /**
+   * @author Eike Stepper
+   */
+  public class Chunk
+  {
+    private int startIndex;
+
+    private Object ids;
+
+    public Chunk(int startIndex)
+    {
+      this.startIndex = startIndex;
+    }
+
+    public Chunk(int startIndex, int size)
+    {
+      this(startIndex);
+      ids = new CDOID[size];
+    }
+
+    public int getStartIndex()
+    {
+      return startIndex;
+    }
+
+    public int size()
+    {
+      if (ids instanceof CDOID)
+      {
+        return 1;
+      }
+
+      return ((CDOID[])ids).length;
+    }
+
+    public CDOID getID(int indexInChunk)
+    {
+      if (ids instanceof CDOID)
+      {
+        if (indexInChunk == 0)
+        {
+          return (CDOID)ids;
+        }
+
+        throw new ArrayIndexOutOfBoundsException(indexInChunk);
+      }
+
+      return ((CDOID[])ids)[indexInChunk];
+    }
+
+    public void addID(int indexInChunk, CDOID id)
+    {
+      if (ids instanceof CDOID)
+      {
+        if (indexInChunk == 0)
+        {
+          ids = id;
+        }
+
+        throw new ArrayIndexOutOfBoundsException(indexInChunk);
+      }
+
+      ((CDOID[])ids)[indexInChunk] = id;
+    }
+  }
+}
diff --git a/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreReader.java b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreReader.java
index 392a6a8626..5dea6eb87c 100644
--- a/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreReader.java
+++ b/plugins/org.eclipse.emf.cdo.server/src/org/eclipse/emf/cdo/server/IStoreReader.java
@@ -15,6 +15,7 @@ import org.eclipse.emf.cdo.internal.protocol.revision.CDORevisionImpl;
 import org.eclipse.emf.cdo.internal.server.StoreUtil;
 import org.eclipse.emf.cdo.protocol.CDOID;
 import org.eclipse.emf.cdo.protocol.model.CDOClassRef;
+import org.eclipse.emf.cdo.protocol.model.CDOFeature;
 import org.eclipse.emf.cdo.protocol.model.CDOPackageInfo;
 import org.eclipse.emf.cdo.protocol.revision.CDORevision;
 
@@ -30,6 +31,8 @@ public interface IStoreReader extends IStoreAccessor
 {
   public ISession getSession();
 
+  public IStoreChunkReader createChunkReader(CDORevision revision, CDOFeature feature);
+
   public Collection<CDOPackageInfo> readPackageInfos();
 
   /**