[485487] Network timeouts can lead to incorrect rollback and stuck session/transaction

https://bugs.eclipse.org/bugs/show_bug.cgi?id=485487

6 files changed, 194 insertions, 56 deletions

diff --git a/plugins/org.eclipse.emf.cdo.tests/META-INF/MANIFEST.MF b/plugins/org.eclipse.emf.cdo.tests/META-INF/MANIFEST.MF
index 84ee8f906e..51fdfbfdf2 100644
--- a/plugins/org.eclipse.emf.cdo.tests/META-INF/MANIFEST.MF
+++ b/plugins/org.eclipse.emf.cdo.tests/META-INF/MANIFEST.MF
@@ -53,6 +53,7 @@ Export-Package: base;version="4.0.200",
  org.eclipse.emf.cdo.tests.config;version="4.0.500",
  org.eclipse.emf.cdo.tests.config.impl;version="4.0.500",
  org.eclipse.emf.cdo.tests.defs;version="4.0.500",
+ org.eclipse.emf.cdo.tests.extra;version="4.0.500",
  org.eclipse.emf.cdo.tests.offline;version="4.0.500",
  org.eclipse.emf.cdo.tests.performance;version="4.0.500",
  org.eclipse.emf.cdo.tests.performance.framework;version="4.0.500",
diff --git a/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/extra/ManyThreads.java b/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/extra/ManyThreads.java
new file mode 100644
index 0000000000..a1a810c409
--- /dev/null
+++ b/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/extra/ManyThreads.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2016 Eike Stepper (Berlin, Germany) and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Eike Stepper - initial API and implementation
+ */
+package org.eclipse.emf.cdo.tests.extra;
+
+import org.eclipse.emf.cdo.eresource.CDOResource;
+import org.eclipse.emf.cdo.session.CDOSession;
+import org.eclipse.emf.cdo.tests.AbstractCDOTest;
+import org.eclipse.emf.cdo.tests.model1.Category;
+import org.eclipse.emf.cdo.tests.model1.Company;
+import org.eclipse.emf.cdo.tests.model1.Product1;
+import org.eclipse.emf.cdo.transaction.CDOTransaction;
+
+import org.eclipse.emf.internal.cdo.transaction.CDOTransactionImpl;
+
+/**
+ * @author Eike Stepper
+ */
+public class ManyThreads extends AbstractCDOTest
+{
+  private static final int COMMITTERS = 250;
+
+  // @Override
+  // public synchronized Map<String, Object> getTestProperties()
+  // {
+  // Map<String, Object> props = super.getTestProperties();
+  // props.put(IRepository.Props.OPTIMISTIC_LOCKING_TIMEOUT, "30000");
+  // return props;
+  // }
+
+  public void testManyParallelCommits() throws Exception
+  {
+    disableConsole();
+
+    final CDOSession session = openSession();
+    CDOTransaction transaction = session.openTransaction();
+    CDOResource resource = transaction.createResource(getResourcePath("test"));
+
+    final Company company = getModel1Factory().createCompany();
+    resource.getContents().add(company);
+
+    Committer[] threads = new Committer[COMMITTERS];
+    for (int i = 0; i < COMMITTERS; i++)
+    {
+      Category category = getModel1Factory().createCategory();
+      category.setName("Category-" + i);
+      company.getCategories().add(category);
+
+      threads[i] = new Committer(session, category, i);
+    }
+
+    transaction.commit();
+    sleep(1000);
+
+    for (Committer thread : threads)
+    {
+      thread.start();
+    }
+
+    for (Committer thread : threads)
+    {
+      thread.join();
+    }
+
+    for (Committer thread : threads)
+    {
+      if (thread.exception != null)
+      {
+        throw thread.exception;
+      }
+    }
+  }
+
+  /**
+   * @author Eike Stepper
+   */
+  private final class Committer extends Thread
+  {
+    private final CDOSession session;
+
+    private Category category;
+
+    private Exception exception;
+
+    public Committer(CDOSession session, Category category, int i)
+    {
+      super("Committer-" + i);
+      this.session = session;
+      this.category = category;
+    }
+
+    @Override
+    public void run()
+    {
+      CDOTransactionImpl transaction = (CDOTransactionImpl)session.openTransaction();
+      // transaction.options().setCommitInfoTimeout(40000);
+      category = transaction.getObject(category);
+
+      Category subCategory = null;
+
+      for (int j = 0; j < 20000; j++)
+      {
+        if (j % 100 == 0)
+        {
+          subCategory = getModel1Factory().createCategory();
+          subCategory.setName("Subcategory-" + j);
+          category.getCategories().add(subCategory);
+        }
+
+        for (int k = 0; k < 10; k++)
+        {
+          Product1 product = getModel1Factory().createProduct1();
+          product.setName("Product-" + k);
+          subCategory.getProducts().add(product);
+        }
+
+        try
+        {
+          transaction.commit();
+        }
+        catch (Exception ex)
+        {
+          exception = ex;
+          return;
+        }
+      }
+    }
+  }
+}
diff --git a/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/UnitManagerMain.java b/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/extra/UnitManagerMain.java
index 33e8c17d58..ded59918ea 100644
--- a/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/UnitManagerMain.java
+++ b/plugins/org.eclipse.emf.cdo.tests/src/org/eclipse/emf/cdo/tests/extra/UnitManagerMain.java
@@ -8,7 +8,7 @@
  * Contributors:
  *    Eike Stepper - initial API and implementation
  */
-package org.eclipse.emf.cdo.tests;
+package org.eclipse.emf.cdo.tests.extra;
 
 import org.eclipse.emf.cdo.common.revision.CDORevision;
 import org.eclipse.emf.cdo.eresource.CDOResource;
diff --git a/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/cdo/transaction/CDOTransaction.java b/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/cdo/transaction/CDOTransaction.java
index 4431348634..f6a3c0f08a 100644
--- a/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/cdo/transaction/CDOTransaction.java
+++ b/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/cdo/transaction/CDOTransaction.java
@@ -232,7 +232,7 @@ public interface CDOTransaction extends CDOView, CDOCommonTransaction, CDOUserTr
     /**
      * @since 4.5
      */
-    public static final long DEFAULT_COMMIT_INFO_TIMEOUT = 10000;
+    public static final long DEFAULT_COMMIT_INFO_TIMEOUT = 60000;
 
     /**
      * Returns the {@link CDOTransaction transaction} of this options object.
diff --git a/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/internal/cdo/view/CDOViewImpl.java b/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/internal/cdo/view/CDOViewImpl.java
index 4087ffe837..0083c16224 100644
--- a/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/internal/cdo/view/CDOViewImpl.java
+++ b/plugins/org.eclipse.emf.cdo/src/org/eclipse/emf/internal/cdo/view/CDOViewImpl.java
@@ -1156,61 +1156,69 @@ public class CDOViewImpl extends AbstractCDOView implements IExecutorServiceProv
 
       try
       {
-        if (getTimeStamp() != UNSPECIFIED_DATE && CDOStateMachine.SWITCHING_TARGET.get() != Boolean.TRUE)
-        {
-          // Don't invalidate historical views unless during a branch point switch.
-          return;
-        }
+        doInvalidateSynced(branch, lastUpdateTime, allChangedObjects, allDetachedObjects, oldRevisions,
+            clearResourcePathCache);
+      }
+      finally
+      {
+        unlockView();
+      }
+    }
+  }
 
-        try
-        {
-          // Also false for FailureCommitInfos (because of branch==null). Only setLastUpdateTime() is called below.
-          if (branch == getBranch())
-          {
-            if (clearResourcePathCache)
-            {
-              clearResourcePathCacheIfNecessary(null);
-            }
+  private void doInvalidateSynced(CDOBranch branch, long lastUpdateTime, List<CDORevisionKey> allChangedObjects,
+      List<CDOIDAndVersion> allDetachedObjects, Map<CDOID, InternalCDORevision> oldRevisions,
+      boolean clearResourcePathCache)
+  {
+    if (getTimeStamp() != UNSPECIFIED_DATE && CDOStateMachine.SWITCHING_TARGET.get() != Boolean.TRUE)
+    {
+      // Don't invalidate historical views unless during a branch point switch.
+      return;
+    }
 
-            List<CDORevisionDelta> deltas = new ArrayList<CDORevisionDelta>();
-            Map<CDOObject, CDORevisionDelta> revisionDeltas = new HashMap<CDOObject, CDORevisionDelta>();
-            Set<CDOObject> detachedObjects = new HashSet<CDOObject>();
+    try
+    {
+      // Also false for FailureCommitInfos (because of branch==null). Only setLastUpdateTime() is called below.
+      if (branch == getBranch())
+      {
+        if (clearResourcePathCache)
+        {
+          clearResourcePathCacheIfNecessary(null);
+        }
 
-            Map<CDOObject, Pair<CDORevision, CDORevisionDelta>> conflicts = invalidate(allChangedObjects,
-                allDetachedObjects, deltas, revisionDeltas, detachedObjects);
-            handleConflicts(lastUpdateTime, conflicts, deltas);
+        List<CDORevisionDelta> deltas = new ArrayList<CDORevisionDelta>();
+        Map<CDOObject, CDORevisionDelta> revisionDeltas = new HashMap<CDOObject, CDORevisionDelta>();
+        Set<CDOObject> detachedObjects = new HashSet<CDOObject>();
 
-            sendInvalidationNotifications(revisionDeltas.keySet(), detachedObjects);
-            fireInvalidationEvent(lastUpdateTime, Collections.unmodifiableMap(revisionDeltas),
-                Collections.unmodifiableSet(detachedObjects));
+        Map<CDOObject, Pair<CDORevision, CDORevisionDelta>> conflicts = invalidate(allChangedObjects,
+            allDetachedObjects, deltas, revisionDeltas, detachedObjects);
+        handleConflicts(lastUpdateTime, conflicts, deltas);
 
-            // Then send the notifications. The deltas could have been modified by the conflict resolvers.
-            if (!deltas.isEmpty() || !detachedObjects.isEmpty())
-            {
-              sendDeltaNotifications(deltas, detachedObjects, oldRevisions);
-            }
+        sendInvalidationNotifications(revisionDeltas.keySet(), detachedObjects);
+        fireInvalidationEvent(lastUpdateTime, Collections.unmodifiableMap(revisionDeltas),
+            Collections.unmodifiableSet(detachedObjects));
 
-            fireAdaptersNotifiedEvent(lastUpdateTime);
-          }
-        }
-        catch (RuntimeException ex)
-        {
-          if (isActive())
-          {
-            fireEvent(new ThrowableEvent(this, ex));
-            throw ex;
-          }
-        }
-        finally
+        // Then send the notifications. The deltas could have been modified by the conflict resolvers.
+        if (!deltas.isEmpty() || !detachedObjects.isEmpty())
         {
-          setLastUpdateTime(lastUpdateTime);
+          sendDeltaNotifications(deltas, detachedObjects, oldRevisions);
         }
+
+        fireAdaptersNotifiedEvent(lastUpdateTime);
       }
-      finally
+    }
+    catch (RuntimeException ex)
+    {
+      if (isActive())
       {
-        unlockView();
+        fireEvent(new ThrowableEvent(this, ex));
+        throw ex;
       }
     }
+    finally
+    {
+      setLastUpdateTime(lastUpdateTime);
+    }
   }
 
   public ExecutorWorkSerializer getInvalidationRunner()
diff --git a/plugins/org.eclipse.net4j.tests/src/org/eclipse/net4j/util/tests/AbstractOMTest.java b/plugins/org.eclipse.net4j.tests/src/org/eclipse/net4j/util/tests/AbstractOMTest.java
index 29758f5a5a..721ca97508 100644
--- a/plugins/org.eclipse.net4j.tests/src/org/eclipse/net4j/util/tests/AbstractOMTest.java
+++ b/plugins/org.eclipse.net4j.tests/src/org/eclipse/net4j/util/tests/AbstractOMTest.java
@@ -183,13 +183,8 @@ public abstract class AbstractOMTest extends TestCase
 
     if (!SUPPRESS_OUTPUT)
     {
-      IOUtil.OUT().println("*******************************************************"); //$NON-NLS-1$
-      Thread.yield();
-      Thread.sleep(2L);
-      IOUtil.ERR().println(this);
-      Thread.yield();
-      Thread.sleep(2L);
-      IOUtil.OUT().println("*******************************************************"); //$NON-NLS-1$
+      IOUtil.ERR().println("*******************************************************\n" + this //$NON-NLS-1$
+          + "\n*******************************************************"); //$NON-NLS-1$
     }
 
     if (!logSetUpAndTearDown())
@@ -202,8 +197,7 @@ public abstract class AbstractOMTest extends TestCase
 
     if (!SUPPRESS_OUTPUT && logSetUpAndTearDown())
     {
-      IOUtil.OUT().println();
-      IOUtil.OUT().println("------------------------ START ------------------------"); //$NON-NLS-1$
+      IOUtil.OUT().println("\n------------------------ START ------------------------"); //$NON-NLS-1$
     }
 
     enableConsole();
@@ -223,8 +217,7 @@ public abstract class AbstractOMTest extends TestCase
 
     if (!SUPPRESS_OUTPUT && logSetUpAndTearDown())
     {
-      IOUtil.OUT().println("------------------------- END -------------------------"); //$NON-NLS-1$
-      IOUtil.OUT().println();
+      IOUtil.OUT().println("------------------------- END -------------------------\n"); //$NON-NLS-1$
     }
 
     try