Bug 578487 - added tests for RefreshJobI20220203-1310

No real functional / behavior code changes, just refactored RefreshJob
so it can be tested and added trivial tests.

Change-Id: I7f1d339824e5b666a9239600e7da406ab6222513
Signed-off-by: Andrey Loskutov <loskutov@gmx.de>
Reviewed-on: https://git.eclipse.org/r/c/platform/eclipse.platform.resources/+/190369
Tested-by: Platform Bot <platform-bot@eclipse.org>

3 files changed, 366 insertions, 10 deletions

diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/refresh/RefreshJob.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/refresh/RefreshJob.java
index 4b762a9e3..39511e08b 100644
--- a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/refresh/RefreshJob.java
+++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/refresh/RefreshJob.java
@@ -30,7 +30,28 @@ import org.eclipse.core.runtime.*;
  * @since 3.0
  */
 public class RefreshJob extends WorkspaceJob {
-	private static final long UPDATE_DELAY = 200;
+
+	/**
+	 * Threshold (in milliseconds) at which the refresh operation is considered to
+	 * be fast enough to increase refresh depth
+	 */
+	public static final int FAST_REFRESH_THRESHOLD = 1000;
+
+	/**
+	 * Threshold (in milliseconds) at which the refresh operation is considered to
+	 * be slow enough to decrease refresh depth
+	 */
+	public static final int SLOW_REFRESH_THRESHOLD = 2000;
+
+	/** Base depth used for refresh */
+	public static final int BASE_REFRESH_DEPTH = 1000;
+
+	/** Number of refresh rounds before updating refresh depth */
+	public static final int DEPTH_INCREASE_STEP = 1000;
+
+	/** Default refresh job delay (in milliseconds) */
+	public static final int UPDATE_DELAY = 200;
+
 	/**
 	 * List of refresh requests. Requests are processed in order from
 	 * the end of the list. Requests can be added to either the beginning
@@ -45,9 +66,28 @@ public class RefreshJob extends WorkspaceJob {
 	 */
 	private PrefixPool pathPrefixHistory, rootPathHistory;
 
+	private final int fastRefreshThreshold;
+	private final int slowRefreshThreshold;
+	private final int baseRefreshDepth;
+	private final int depthIncreaseStep;
+	private final int updateDelay;
+
 	public RefreshJob() {
+		this(FAST_REFRESH_THRESHOLD, SLOW_REFRESH_THRESHOLD, BASE_REFRESH_DEPTH, DEPTH_INCREASE_STEP, UPDATE_DELAY);
+	}
+
+	/**
+	 * This method is protected for tests
+	 */
+	protected RefreshJob(int fastRefreshThreshold, int slowRefreshThreshold, int baseRefreshDepth,
+			int depthIncreaseStep, int updateDelay) {
 		super(Messages.refresh_jobName);
-		fRequests = new ArrayList<>(1);
+		this.fRequests = new ArrayList<>(1);
+		this.fastRefreshThreshold = fastRefreshThreshold;
+		this.slowRefreshThreshold = slowRefreshThreshold;
+		this.baseRefreshDepth = baseRefreshDepth;
+		this.depthIncreaseStep = depthIncreaseStep;
+		this.updateDelay = updateDelay;
 	}
 
 	/**
@@ -72,7 +112,9 @@ public class RefreshJob extends WorkspaceJob {
 
 	private synchronized void addRequests(List<IResource> list) {
 		//add requests to the end of the queue
-		fRequests.addAll(0, list);
+		if (!list.isEmpty()) {
+			fRequests.addAll(0, list);
+		}
 	}
 
 	@Override
@@ -84,7 +126,7 @@ public class RefreshJob extends WorkspaceJob {
 	 * This method adds all members at the specified depth from the resource
 	 * to the provided list.
 	 */
-	private List<IResource> collectChildrenToDepth(IResource resource, ArrayList<IResource> children, int depth) {
+	protected List<IResource> collectChildrenToDepth(IResource resource, ArrayList<IResource> children, int depth) {
 		if (resource.getType() == IResource.FILE)
 			return children;
 		IResource[] members;
@@ -141,7 +183,7 @@ public class RefreshJob extends WorkspaceJob {
 		if (resource == null)
 			return;
 		addRequest(resource);
-		schedule(UPDATE_DELAY);
+		schedule(updateDelay);
 	}
 
 	@Override
@@ -163,20 +205,26 @@ public class RefreshJob extends WorkspaceJob {
 					subMonitor.setWorkRemaining(Math.max(fRequests.size(), 100));
 					refreshCount++;
 					long refreshTime = -System.currentTimeMillis();
-					toRefresh.refreshLocal(1000 + depth, subMonitor.split(1));
+					toRefresh.refreshLocal(baseRefreshDepth + depth, subMonitor.split(1));
 					refreshTime += System.currentTimeMillis();
 					if (refreshTime > longestRefresh)
 						longestRefresh = refreshTime;
 					//show occasional progress
-					if (refreshCount % 1000 == 0) {
+					if (refreshCount % depthIncreaseStep == 0) {
 						//be polite to other threads (no effect on some platforms)
 						Thread.yield();
 						//throttle depth if it takes too long
-						if (longestRefresh > 2000 && depth > 1) {
+						if (longestRefresh > slowRefreshThreshold && depth > 1) {
 							depth = 1;
+							if (Policy.DEBUG_AUTO_REFRESH) {
+								Policy.debug(RefreshManager.DEBUG_PREFIX + " decreased refresh depth to: " + depth); //$NON-NLS-1$
+							}
 						}
-						if (longestRefresh < 1000) {
+						if (longestRefresh < fastRefreshThreshold) {
 							depth *= 2;
+							if (Policy.DEBUG_AUTO_REFRESH) {
+								Policy.debug(RefreshManager.DEBUG_PREFIX + " increased refresh depth to: " + depth); //$NON-NLS-1$
+							}
 						}
 						longestRefresh = 0;
 					}
diff --git a/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/AllTests.java b/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/AllTests.java
index dff523a04..5d846c0d4 100644
--- a/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/AllTests.java
+++ b/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/AllTests.java
@@ -20,6 +20,6 @@ import org.junit.runners.Suite;
  * Runs all tests in this package.
  */
 @RunWith(Suite.class)
-@Suite.SuiteClasses({ RefreshProviderTest.class })
+@Suite.SuiteClasses({ RefreshProviderTest.class, RefreshJobTest.class })
 public class AllTests {
 }
diff --git a/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/RefreshJobTest.java b/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/RefreshJobTest.java
new file mode 100644
index 000000000..ec88b9aaf
--- /dev/null
+++ b/tests/org.eclipse.core.tests.resources/src/org/eclipse/core/tests/resources/refresh/RefreshJobTest.java
@@ -0,0 +1,308 @@
+/*******************************************************************************
+ *  Copyright (c) 2022 Andrey Loskutov <loskutov@gmx.de> and others.
+ *
+ *  This program and the accompanying materials
+ *  are made available under the terms of the Eclipse Public License 2.0
+ *  which accompanies this distribution, and is available at
+ *  https://www.eclipse.org/legal/epl-2.0/
+ *
+ *  SPDX-License-Identifier: EPL-2.0
+ *
+ *  Contributors:
+ *     Andrey Loskutov <loskutov@gmx.de> - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.core.tests.resources.refresh;
+
+import static org.eclipse.core.internal.refresh.RefreshJob.BASE_REFRESH_DEPTH;
+import static org.eclipse.core.internal.refresh.RefreshJob.DEPTH_INCREASE_STEP;
+import static org.eclipse.core.internal.refresh.RefreshJob.FAST_REFRESH_THRESHOLD;
+import static org.eclipse.core.internal.refresh.RefreshJob.SLOW_REFRESH_THRESHOLD;
+import static org.eclipse.core.internal.refresh.RefreshJob.UPDATE_DELAY;
+import static org.junit.Assert.assertArrayEquals;
+
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.eclipse.core.internal.refresh.RefreshJob;
+import org.eclipse.core.internal.refresh.RefreshManager;
+import org.eclipse.core.internal.resources.Workspace;
+import org.eclipse.core.resources.*;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IPath;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
+import org.eclipse.core.tests.resources.ResourceTest;
+import org.eclipse.core.tests.resources.TestUtil;
+
+/**
+ * Tests for RefreshJob
+ */
+public class RefreshJobTest extends ResourceTest {
+
+	private boolean defaultRefresh;
+	private boolean shouldResetDefault;
+
+	int fastRefreshThreshold = FAST_REFRESH_THRESHOLD;
+	int slowRefreshThreshold = SLOW_REFRESH_THRESHOLD;
+	int baseRefreshDepth = BASE_REFRESH_DEPTH;
+	int depthIncreaseStep = DEPTH_INCREASE_STEP;
+	int updateDelay = UPDATE_DELAY;
+	private RefreshJob originalJob;
+
+	@Override
+	protected void setUp() throws Exception {
+		super.setUp();
+		IEclipsePreferences prefs = getPrefs();
+		defaultRefresh = prefs.getBoolean(ResourcesPlugin.PREF_AUTO_REFRESH, false);
+		if (defaultRefresh) {
+			prefs.putBoolean(ResourcesPlugin.PREF_AUTO_REFRESH, false);
+			shouldResetDefault = true;
+		}
+		TestUtil.waitForJobs("setup", 100, 1000);
+		// we don't want to wait extra time
+		updateDelay = 0;
+	}
+
+	@Override
+	protected void tearDown() throws Exception {
+		restoreRefreshJob();
+		if (shouldResetDefault) {
+			getPrefs().putBoolean(ResourcesPlugin.PREF_AUTO_REFRESH, defaultRefresh);
+		}
+		super.tearDown();
+	}
+
+	private IEclipsePreferences getPrefs() {
+		return InstanceScope.INSTANCE.getNode(ResourcesPlugin.PI_RESOURCES);
+	}
+
+	/**
+	 * Test to ensure that there is no endless loop on refresh
+	 *
+	 * XXX test is disabled because it demonstrated endless loop from
+	 * https://bugs.eclipse.org/bugs/show_bug.cgi?id=578487
+	 */
+	public void XtestBug578487_refreshLoop() throws Exception {
+		String name = "testBug578487_refreshLoop";
+		int minDepth = 0;
+		int maxDepth = 2;
+
+		int filesCount = 0;
+		// 9 dirs & 2 depth & 2 depthIncreaseStep == hang
+		int directoriesCount = 9;
+		int createDepth = 2;
+
+		depthIncreaseStep = 2;
+		fastRefreshThreshold = Integer.MAX_VALUE / 2;
+		slowRefreshThreshold = Integer.MAX_VALUE;
+		baseRefreshDepth = BASE_REFRESH_DEPTH;
+		runtest(name, minDepth, maxDepth, directoriesCount, filesCount, createDepth);
+	}
+
+	/**
+	 * Just a trivial test that few directories can be refreshed with default
+	 * settings and default max depth of 2
+	 */
+	public void testBasicRefresh() throws Exception {
+		String name = "testBasicRefresh";
+		int minDepth = 0;
+		int maxDepth = 2;
+
+		int directoriesCount = 3;
+		int filesCount = 1;
+		int createDepth = 2;
+
+		runtest(name, minDepth, maxDepth, directoriesCount, filesCount, createDepth);
+	}
+
+	/**
+	 * Test that few directories can be refreshed with max depth of 16 (simulating a
+	 * very fast file system)
+	 */
+	public void testFastRefresh() throws Exception {
+		String name = "testFastRefresh";
+		int minDepth = 0;
+		int maxDepth = 16;
+
+		int directoriesCount = 3;
+		int filesCount = 1;
+		int createDepth = 2;
+
+		depthIncreaseStep = 2;
+		fastRefreshThreshold = Integer.MAX_VALUE / 2;
+		slowRefreshThreshold = Integer.MAX_VALUE;
+		baseRefreshDepth = BASE_REFRESH_DEPTH;
+
+		runtest(name, minDepth, maxDepth, directoriesCount, filesCount, createDepth);
+	}
+
+	/**
+	 * Test that few directories can be refreshed with max depth of 1 (simulating a
+	 * very slow file system)
+	 */
+	public void testSlowRefresh() throws Exception {
+		String name = "testSlowRefresh";
+		int minDepth = 0;
+		int maxDepth = 1;
+
+		int directoriesCount = 3;
+		int filesCount = 1;
+		int createDepth = 2;
+
+		depthIncreaseStep = 1;
+		fastRefreshThreshold = Integer.MIN_VALUE;
+		slowRefreshThreshold = Integer.MIN_VALUE;
+		baseRefreshDepth = BASE_REFRESH_DEPTH;
+
+		runtest(name, minDepth, maxDepth, directoriesCount, filesCount, createDepth);
+	}
+
+	private void runtest(String name, int minDepth, int maxDepth, int directoriesCount, int filesCount,
+			int createDepth) throws Exception, CoreException {
+		try {
+			IProject project = createProject(name);
+			IPath projectRoot = project.getLocation();
+			project.close(null);
+
+			AtomicInteger result = new AtomicInteger(0);
+			createDirectoriesViaFileIo(projectRoot.toFile().toPath(), directoriesCount, filesCount, createDepth,
+					result);
+			assertTrue("Expect to generate some files", result.get() > 0);
+			System.out.println("\nTest " + name + " generated " + result + " files");
+
+			project.open(null);
+			TestRefreshJob refreshJob = createAndReplaceDefaultJob();
+			refreshJob.refresh(project);
+			waitForRefresh();
+			assertAllResourcesRefreshed(project, refreshJob);
+			assertDepth(refreshJob, minDepth, maxDepth);
+		} finally {
+			deleteProject(name);
+		}
+	}
+
+	private void assertDepth(TestRefreshJob refreshJob, int minDepth, int maxDepth) {
+		assertEquals("Unexpected min depth", minDepth, refreshJob.minDepth);
+		assertEquals("Unexpected max depth", maxDepth, refreshJob.maxDepth);
+	}
+
+	private void assertAllResourcesRefreshed(IProject project, TestRefreshJob refreshJob) throws Exception {
+		Set<IResource> resources = new LinkedHashSet<>(refreshJob.visitedResources);
+		project.refreshLocal(IResource.DEPTH_INFINITE, null);
+		Set<IResource> missing = new LinkedHashSet<>();
+		Set<IResource> visited = new LinkedHashSet<>();
+		project.accept(new IResourceVisitor() {
+			@Override
+			public boolean visit(IResource resource) throws CoreException {
+				if (resource.getType() == IResource.FILE) {
+					return true;
+				}
+				visited.add(resource);
+				if (!resources.contains(resource)) {
+					missing.add(resource);
+				}
+				return true;
+			}
+		});
+		assertArrayEquals("Resources not refreshed", new IResource[0], missing.toArray());
+		assertFalse("Projects should be not empty", visited.isEmpty());
+		assertFalse("No resources refreshed", resources.isEmpty());
+	}
+
+	private TestRefreshJob createAndReplaceDefaultJob() throws Exception {
+		TestRefreshJob job = new TestRefreshJob(fastRefreshThreshold, slowRefreshThreshold, baseRefreshDepth,
+				depthIncreaseStep, updateDelay);
+
+		RefreshManager refreshManager = ((Workspace) getWorkspace()).getRefreshManager();
+		Field field = RefreshManager.class.getDeclaredField("refreshJob");
+		field.setAccessible(true);
+		originalJob = (RefreshJob) field.get(refreshManager);
+		field.set(refreshManager, job);
+		return job;
+	}
+
+	private void restoreRefreshJob() throws Exception {
+		RefreshManager refreshManager = ((Workspace) getWorkspace()).getRefreshManager();
+		Field field = RefreshManager.class.getDeclaredField("refreshJob");
+		field.setAccessible(true);
+		field.set(refreshManager, originalJob);
+	}
+
+	private void createDirectoriesViaFileIo(Path root, int directoriesCount, int filesCount, int createDepth,
+			AtomicInteger result)
+			throws Exception {
+		if (createDepth <= 0) {
+			return;
+		}
+		List<Path> dirs = new ArrayList<>();
+		for (int i = 0; i < directoriesCount; i++) {
+			Path dir = Files.createDirectory(root.resolve("dir_" + i));
+			result.incrementAndGet();
+			dirs.add(dir);
+			for (int j = 0; j < filesCount; j++) {
+				Files.createFile(dir.resolve("file_" + j));
+				result.incrementAndGet();
+			}
+		}
+		createDepth--;
+		directoriesCount--;
+		filesCount--;
+		for (Path dir : dirs) {
+			createDirectoriesViaFileIo(dir, directoriesCount, filesCount, createDepth, result);
+		}
+	}
+
+	private IProject createProject(String name) throws Exception {
+		IWorkspaceRoot root = getWorkspaceRoot();
+		assertFalse(deleteProject(name).isAccessible());
+		IProject project = root.getProject(name);
+		project.create(null);
+		project.open(null);
+		return project;
+	}
+
+
+	private static IWorkspaceRoot getWorkspaceRoot() {
+		return getWorkspace().getRoot();
+	}
+
+	private static IProject deleteProject(String name) throws Exception {
+		IProject pro = getWorkspaceRoot().getProject(name);
+		if (pro.exists()) {
+			pro.delete(true, true, null);
+		}
+		return pro;
+	}
+
+	class TestRefreshJob extends RefreshJob {
+
+		int maxDepth;
+		int minDepth;
+		Set<IResource> visitedResources = new LinkedHashSet<>();
+
+		protected TestRefreshJob(int fastRefreshThreshold, int slowRefreshThreshold, int baseRefreshDepth,
+				int depthIncreaseStep, int updateDelay) {
+			super(fastRefreshThreshold, slowRefreshThreshold, baseRefreshDepth,
+					depthIncreaseStep, updateDelay);
+		}
+
+		@Override
+		protected List<IResource> collectChildrenToDepth(IResource resource, ArrayList<IResource> children, int depth) {
+			System.out.println("collectChildrenToDepth " + depth + ":" + resource);
+			List<IResource> list = super.collectChildrenToDepth(resource, children, depth);
+			visitedResources.add(resource);
+			visitedResources.addAll(list);
+			if (maxDepth < depth) {
+				maxDepth = depth;
+			}
+			if (minDepth > depth) {
+				minDepth = depth;
+			}
+			return list;
+		}
+
+	}
+
+}