diff options
author | Joerg Kubitz | 2021-09-01 10:28:55 +0000 |
---|---|---|
committer | Gayan Perera | 2021-10-12 17:17:54 +0000 |
commit | 9525f3eec57b403f918771025f4280bba56b1221 (patch) | |
tree | 466d27c5f914442abb18619110d43fa9fe0c48eb | |
parent | dd4b2f42ec36c54ae2ee0e63414fc8b98a383188 (diff) | |
download | eclipse.jdt.core-9525f3eec57b403f918771025f4280bba56b1221.tar.gz eclipse.jdt.core-9525f3eec57b403f918771025f4280bba56b1221.tar.xz eclipse.jdt.core-9525f3eec57b403f918771025f4280bba56b1221.zip |
Bug 575733 - refactored Scanner/PublicScanner char deduplication
Scanner.<init>() [initial filling of this.charArray_length] took more
time then all the optimizedCurrentTokenSource* methods together.
This was a hotspot when (for example xtext) created a new scanner for
each class.
Now use a single shared deduplicationpool per thread - which only needs
to be initialized once per thread.
+also deduplicate all single Ascii chars.
Change-Id: I3ee49cb1a4b0543c82d9c0e2214328aa12613309
Signed-off-by: Joerg Kubitz <jkubitz-eclipse@gmx.de>
Reviewed-on: https://git.eclipse.org/r/c/jdt/eclipse.jdt.core/+/184844
Tested-by: JDT Bot <jdt-bot@eclipse.org>
Reviewed-by: Gayan Perera <gayanper@gmail.com>
5 files changed, 484 insertions, 603 deletions
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/CharDeduplicationTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/CharDeduplicationTest.java new file mode 100644 index 0000000000..92fa0a2ae2 --- /dev/null +++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/CharDeduplicationTest.java @@ -0,0 +1,187 @@ +/******************************************************************************* + * Copyright (c) 2021 jkubitz and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * jkubitz - initial API and implementation + *******************************************************************************/ +package org.eclipse.jdt.core.tests.compiler; + +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import org.eclipse.jdt.core.tests.junit.extension.TestCase; +import org.eclipse.jdt.internal.compiler.util.CharDeduplication; + +import junit.framework.Test; +import junit.framework.TestSuite; + +public class CharDeduplicationTest extends TestCase { + + public CharDeduplicationTest(String testName) { + super(testName); + } + + public static Test suite() { + + TestSuite suite = new TestSuite(CharDeduplicationTest.class.getPackageName()); + suite.addTest(new TestSuite(CharDeduplicationTest.class)); + return suite; + } + + public void testDeduplication() { + for (int i = 0; i < 3; i++) { + assertDeduplication("a"); + // .. + assertDeduplication("z"); + + assertDeduplication("12"); + assertDeduplication("123"); + assertDeduplication("1234"); + assertDeduplication("12345"); + assertDeduplication("123456"); + assertNoDeduplication("1234567"); + + // new: + assertDeduplication("0"); // illegal identifier - but who cares. + assertDeduplication("A"); // why not? + assertDeduplication("$"); // legal + assertDeduplication("_"); // note that "_" may become more common after JEP 302 as keyword for unused + // lambda parameter! + assertDeduplication("" + (char) 0); + // .. + assertDeduplication("" + (char) 127); + assertNoDeduplication("" + (char) 128); // non-Ascii + } + } + + public void testDeduplicationMid() { + String text = "abcdefghijklmn"; + for (int start = 0; start < text.length(); start++) { + for (int end = start; end < text.length(); end++) { + assertDedup(text, (end - start) <= CharDeduplication.OPTIMIZED_LENGTH, start, end); + } + } + } + + @SuppressWarnings("deprecation") + public void testDeduplicationTableSize() { + CharDeduplication deduplication = CharDeduplication.getThreadLocalInstance(); + deduplication.reset();// to test the overflow we need to start empty + for (int overload = 0; overload < 3; overload++) { + HashMap<Integer, char[]> expecteds = new HashMap<>(); + for (int i = 0; i < CharDeduplication.TABLE_SIZE + overload; i++) { + int numberWithFixedLength = 10000 + i; + String string = "" + numberWithFixedLength; + char[] a = string.toCharArray(); + char[] expected = deduplication.sharedCopyOfRange(a, 0, a.length); + expecteds.put(i, expected); + } + for (int t = 0; t < 2; t++) { + for (int i = 0; i < expecteds.size(); i++) { + char[] expected = expecteds.get(i); + char[] other = String.valueOf(expected).toCharArray(); + if (overload == 0 || i > 0) { + assertDedup(true, 0, expected.length, expected, other); + } else { + // situation after table overflow: + char[] actual = deduplication.sharedCopyOfRange(other, 0, expected.length); + // both actual == expected or actual != expected may happen + // but we can assert that the next deduplication works again: + char[] other2 = String.valueOf(expected).toCharArray(); + assertDedup(true, 0, expected.length, actual, other2); + } + } + } + } + } + + public static void main(String[] args) { + CharDeduplicationTest test=new CharDeduplicationTest(""); + System.out.println("min= ~"+ LongStream.range(0, 100).map(t->test.runPerformanceTest()).min()); + // min= ~0.36sec + } + + public long runPerformanceTest() { + long nanoTime = System.nanoTime(); + CharDeduplication deduplication = CharDeduplication.getThreadLocalInstance(); + for (int j = 0; j < 100; j++) { + for (int i = 0; i < 100_000; i++) { + String hexString = Integer.toHexString(i); + char[] chars = hexString.toCharArray(); + deduplication.sharedCopyOfRange(chars, 0, chars.length); + } + } + long nanoTime2 = System.nanoTime(); + long durationNanos = nanoTime2 - nanoTime; + System.out.println(durationNanos); + return durationNanos; + } + + public void testAll() { + testDeduplication(); + testDeduplicationMid(); + testDeduplicationTableSize(); + } + + public void testMultithreaded() throws Exception { + int nThreads = 8; + List<FutureTask<Object>> tasks = IntStream.range(0, nThreads * 2).mapToObj(i -> new FutureTask<Object>(() -> { + testAll(); + return null; + })).collect(Collectors.toList()); + ExecutorService executor = Executors.newFixedThreadPool(nThreads); + tasks.forEach(executor::submit); + for (FutureTask<Object> task : tasks) { + try { + task.get(); + } catch (Exception e) { + throw new AssertionError(e); + } + } + executor.shutdownNow(); + } + + private void assertDeduplication(String string) { + assertDedup(string, true, 0, string.length()); + } + + private void assertNoDeduplication(String string) { + assertDedup(string, false, 0, string.length()); + } + + private void assertDedup(String string, boolean same, int startPosition, int end) { + char[] a = string.toCharArray(); + char[] b = string.toCharArray(); + assertNotSame(a, b); + CharDeduplication deduplication = CharDeduplication.getThreadLocalInstance(); + char[] expected = deduplication.sharedCopyOfRange(a, startPosition, end); + assertDedup(same, startPosition, end, expected, b); + } + + private char[] assertDedup(boolean same, int startPosition, int end, char[] expected, char[] other) { + assertNotSame(expected, other); + CharDeduplication deduplication = CharDeduplication.getThreadLocalInstance(); + char[] actual = deduplication.sharedCopyOfRange(other, startPosition, end); + String state = "expected=" + String.valueOf(expected) + ", actual=" + String.valueOf(actual); + if (same) { + assertSame(state, expected, actual); + } else { + assertNotSame("Expected different instances. But thats not a requirement but an implementation detail test:" + + state, expected, actual); + } + return actual; + } +} diff --git a/org.eclipse.jdt.core.tests.model/src/org/eclipse/jdt/core/tests/model/AllJavaModelTests.java b/org.eclipse.jdt.core.tests.model/src/org/eclipse/jdt/core/tests/model/AllJavaModelTests.java index 8590fb38bf..1d514561ab 100644 --- a/org.eclipse.jdt.core.tests.model/src/org/eclipse/jdt/core/tests/model/AllJavaModelTests.java +++ b/org.eclipse.jdt.core.tests.model/src/org/eclipse/jdt/core/tests/model/AllJavaModelTests.java @@ -18,6 +18,7 @@ package org.eclipse.jdt.core.tests.model; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import org.eclipse.jdt.core.tests.compiler.CharDeduplicationTest; import org.eclipse.jdt.core.tests.compiler.map.CharArrayMapperTest; import org.eclipse.jdt.core.tests.junit.extension.TestCase; @@ -229,6 +230,8 @@ private static Class[] getAllTestClasses() { JavaModelManagerTests.class, CharArrayMapperTest.class, + + CharDeduplicationTest.class, }; Class[] deprecatedClasses = getDeprecatedJDOMTestClasses(); diff --git a/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/parser/Scanner.java b/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/parser/Scanner.java index 4b3f9bad12..a3e9af1686 100644 --- a/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/parser/Scanner.java +++ b/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/parser/Scanner.java @@ -31,6 +31,7 @@ import org.eclipse.jdt.internal.compiler.impl.CompilerOptions; import org.eclipse.jdt.internal.compiler.impl.JavaFeature; import org.eclipse.jdt.internal.compiler.problem.DefaultProblemFactory; import org.eclipse.jdt.internal.compiler.problem.ProblemReporter; +import org.eclipse.jdt.internal.compiler.util.CharDeduplication; import org.eclipse.jdt.internal.compiler.util.Util; /** @@ -154,41 +155,6 @@ public class Scanner implements TerminalTokens { public static final String INVALID_UNDERSCORE = "Invalid_Underscore"; //$NON-NLS-1$ public static final String UNDERSCORES_IN_LITERALS_NOT_BELOW_17 = "Underscores_In_Literals_Not_Below_17"; //$NON-NLS-1$ - //----------------optimized identifier managment------------------ - static final char[] charArray_a = new char[] {'a'}, - charArray_b = new char[] {'b'}, - charArray_c = new char[] {'c'}, - charArray_d = new char[] {'d'}, - charArray_e = new char[] {'e'}, - charArray_f = new char[] {'f'}, - charArray_g = new char[] {'g'}, - charArray_h = new char[] {'h'}, - charArray_i = new char[] {'i'}, - charArray_j = new char[] {'j'}, - charArray_k = new char[] {'k'}, - charArray_l = new char[] {'l'}, - charArray_m = new char[] {'m'}, - charArray_n = new char[] {'n'}, - charArray_o = new char[] {'o'}, - charArray_p = new char[] {'p'}, - charArray_q = new char[] {'q'}, - charArray_r = new char[] {'r'}, - charArray_s = new char[] {'s'}, - charArray_t = new char[] {'t'}, - charArray_u = new char[] {'u'}, - charArray_v = new char[] {'v'}, - charArray_w = new char[] {'w'}, - charArray_x = new char[] {'x'}, - charArray_y = new char[] {'y'}, - charArray_z = new char[] {'z'}; - - static final char[] initCharArray = - new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'}; - static final int TableSize = 30, InternalTableSize = 6; //30*6 =210 entries - - public static final int OptimizedLength = 6; - public /*static*/ final char[][][][] charArray_length = - new char[OptimizedLength - 1][TableSize][InternalTableSize][]; // support for detecting non-externalized string literals public static final char[] TAG_PREFIX= "//$NON-NLS-".toCharArray(); //$NON-NLS-1$ public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length; @@ -209,20 +175,6 @@ public class Scanner implements TerminalTokens { // generic support public boolean returnOnlyGreater = false; - /*static*/ { - for (int i = 0; i < OptimizedLength - 1; i++) { - for (int j = 0; j < TableSize; j++) { - for (int k = 0; k < InternalTableSize; k++) { - this.charArray_length[i][j][k] = initCharArray; - } - } - } - } - /*static*/ int newEntry2 = 0, - newEntry3 = 0, - newEntry4 = 0, - newEntry5 = 0, - newEntry6 = 0; public boolean insideRecovery = false; int lookBack[] = new int[2]; // fall back to spring forward. protected int nextToken = TokenNameNotAToken; // allows for one token push back, only the most recent token can be reliably ungotten. @@ -249,6 +201,8 @@ public class Scanner implements TerminalTokens { //Java 15 - first _ keyword appears Map<String, Integer> _Keywords = null; + private CharDeduplication deduplication = CharDeduplication.getThreadLocalInstance(); + public Scanner() { this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/); } @@ -506,24 +460,9 @@ public char[] getCurrentIdentifierSource() { } int length = this.currentPosition - this.startPosition; if (length == this.eofPosition) return this.source; - switch (length) { // see OptimizedLength - case 1 : - return optimizedCurrentTokenSource1(); - case 2 : - return optimizedCurrentTokenSource2(); - case 3 : - return optimizedCurrentTokenSource3(); - case 4 : - return optimizedCurrentTokenSource4(); - case 5 : - return optimizedCurrentTokenSource5(); - case 6 : - return optimizedCurrentTokenSource6(); - } - char[] result = new char[length]; - System.arraycopy(this.source, this.startPosition, result, 0, length); - return result; + return this.deduplication.sharedCopyOfRange(this.source, this.startPosition, this.currentPosition); } + public int getCurrentTokenEndPosition(){ return this.currentPosition - 1; } @@ -2772,241 +2711,6 @@ public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException { return CharOperation.isWhitespace(this.currentCharacter); } -final char[] optimizedCurrentTokenSource1() { - //return always the same char[] build only once - - //optimization at no speed cost of 99.5 % of the singleCharIdentifier - char charOne = this.source[this.startPosition]; - switch (charOne) { - case 'a' : - return charArray_a; - case 'b' : - return charArray_b; - case 'c' : - return charArray_c; - case 'd' : - return charArray_d; - case 'e' : - return charArray_e; - case 'f' : - return charArray_f; - case 'g' : - return charArray_g; - case 'h' : - return charArray_h; - case 'i' : - return charArray_i; - case 'j' : - return charArray_j; - case 'k' : - return charArray_k; - case 'l' : - return charArray_l; - case 'm' : - return charArray_m; - case 'n' : - return charArray_n; - case 'o' : - return charArray_o; - case 'p' : - return charArray_p; - case 'q' : - return charArray_q; - case 'r' : - return charArray_r; - case 's' : - return charArray_s; - case 't' : - return charArray_t; - case 'u' : - return charArray_u; - case 'v' : - return charArray_v; - case 'w' : - return charArray_w; - case 'x' : - return charArray_x; - case 'y' : - return charArray_y; - case 'z' : - return charArray_z; - default : - return new char[] {charOne}; - } -} -final char[] optimizedCurrentTokenSource2() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0 , c1; - int hash = (((c0=src[start]) << 6) + (c1=src[start+1])) % TableSize; - char[][] table = this.charArray_length[0][hash]; - int i = this.newEntry2; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry2; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[2], 0, 2); - //newIdentCount++; - return table[this.newEntry2 = max] = r; //(r = new char[] {c0, c1}); -} -final char[] optimizedCurrentTokenSource3() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1=src[start+1], c2; - int hash = (((c0=src[start])<< 6) + (c2=src[start+2])) % TableSize; -// int hash = ((c0 << 12) + (c1<< 6) + c2) % TableSize; - char[][] table = this.charArray_length[1][hash]; - int i = this.newEntry3; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry3; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[3], 0, 3); - //newIdentCount++; - return table[this.newEntry3 = max] = r; //(r = new char[] {c0, c1, c2}); -} -final char[] optimizedCurrentTokenSource4() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1 = src[start+1], c2, c3 = src[start+3]; - int hash = (((c0=src[start]) << 6) + (c2=src[start+2])) % TableSize; -// int hash = (int) (((((long) c0) << 18) + (c1 << 12) + (c2 << 6) + c3) % TableSize); - char[][] table = this.charArray_length[2][hash]; - int i = this.newEntry4; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry4; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[4], 0, 4); - //newIdentCount++; - return table[this.newEntry4 = max] = r; //(r = new char[] {c0, c1, c2, c3}); -} -final char[] optimizedCurrentTokenSource5() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1 = src[start+1], c2, c3 = src[start+3], c4; - int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize; -// int hash = (int) (((((long) c0) << 24) + (((long) c1) << 18) + (c2 << 12) + (c3 << 6) + c4) % TableSize); - char[][] table = this.charArray_length[3][hash]; - int i = this.newEntry5; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry5; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[5], 0, 5); - //newIdentCount++; - return table[this.newEntry5 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4}); -} -final char[] optimizedCurrentTokenSource6() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1 = src[start+1], c2, c3 = src[start+3], c4, c5 = src[start+5]; - int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize; -// int hash = (int)(((((long) c0) << 32) + (((long) c1) << 24) + (((long) c2) << 18) + (c3 << 12) + (c4 << 6) + c5) % TableSize); - char[][] table = this.charArray_length[4][hash]; - int i = this.newEntry6; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4]) - && (c5 == charArray[5])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry6; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4]) - && (c5 == charArray[5])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[6], 0, 6); - //newIdentCount++; - return table[this.newEntry6 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4, c5}); -} public boolean isInModuleDeclaration() { return this.fakeInModule || this.insideModuleInfo || (this.activeParser != null ? this.activeParser.isParsingModuleDeclaration() : false); diff --git a/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/util/CharDeduplication.java b/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/util/CharDeduplication.java new file mode 100644 index 0000000000..50dc2fa909 --- /dev/null +++ b/org.eclipse.jdt.core/compiler/org/eclipse/jdt/internal/compiler/util/CharDeduplication.java @@ -0,0 +1,284 @@ +/******************************************************************************* + * Copyright (c) 2021 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Joerg Kubitz - threadlocal refactoring, all ASCII chars + * - (copied content from PublicScanner.java / Scanner.java) + *******************************************************************************/ +package org.eclipse.jdt.internal.compiler.util; + +import java.lang.ref.SoftReference; +import java.util.Arrays; +import java.util.function.Supplier; + +public class CharDeduplication { + + // ----- immutable static part (thread safe): ---- + + static final char[] ASCII_CHARS[] = new char[128][]; + static { + for (int i = 0; i < ASCII_CHARS.length; i++) { + ASCII_CHARS[i] = new char[] { (char) i }; + } + } + public static final int TABLE_SIZE = 30; // XXX thats not a prime -> bad for hashing, nor a power of 2 -> expensive + // modulo computation + public static final int INTERNAL_TABLE_SIZE = 6; // 30*6 =180 entries + + public static final int OPTIMIZED_LENGTH = 6; + + private final static char[] CHAR_ARRAY0 = new char[0]; + + /** avoid OOME by additional CharDeduplication memory **/ + static final class CacheReference<T> { + private SoftReference<T> reference; + private final Supplier<? extends T> supplier; + + CacheReference(Supplier<? extends T> supplier) { + this.supplier = supplier; + this.reference = new SoftReference<>(supplier.get()); + } + + T get() { + T referent = this.reference.get(); + if (referent == null) { + referent = this.supplier.get(); + this.reference = new SoftReference<>(referent); + } + return referent; + } + } + + private final static ThreadLocal<CacheReference<CharDeduplication>> mutableCache = ThreadLocal.withInitial(()->new CacheReference<>(CharDeduplication::new)); + + private static final char[] optimizedCurrentTokenSource1(char[] source, int startPosition) { + // optimization at no speed cost of 99.5 % of the singleCharIdentifier + char charOne = source[startPosition]; + if (charOne < ASCII_CHARS.length) { + return ASCII_CHARS[charOne]; + } + return new char[] { charOne }; + } + + /** @return an instance that is *not* thread safe. To be used in a single thread only. **/ + public static CharDeduplication getThreadLocalInstance() { + return mutableCache.get().get(); + } + + // ----- mutable non-static part (not thread safe!): ---- + + /** single threaded only **/ + public final char[][][][] charArray_length = new char[OPTIMIZED_LENGTH - 1][TABLE_SIZE][INTERNAL_TABLE_SIZE][]; + + int newEntry2 = 0; + int newEntry3 = 0; + int newEntry4 = 0; + int newEntry5 = 0; + int newEntry6 = 0; + + private CharDeduplication() { + init(); + } + + private void init() { + for (int i = 0; i < OPTIMIZED_LENGTH - 1; i++) { + final char[] initCharArray = new char[i + 2]; + for (int j = 0; j < TABLE_SIZE; j++) { + for (int k = 0; k < INTERNAL_TABLE_SIZE; k++) { + this.charArray_length[i][j][k] = initCharArray; + } + } + } + } + + /** public for test purpose only **/ + @Deprecated + public void reset() { + init(); + } + + /** + * like Arrays.copyOfRange(source, from, to) but returns a cached instance of the former result if + * available + * + * @param from + * start index (inclusive) + * @param to + * end index (exclusive) + * @return source[from..to-1] + * @see java.util.Arrays#copyOfRange(char[], int, int) + **/ + public char[] sharedCopyOfRange(char[] source, int from, int to) { + int length = to - from; + switch (length) { // see OptimizedLength + case 1: + return optimizedCurrentTokenSource1(source, from); + case 2: + return optimizedCurrentTokenSource2(source, from); + case 3: + return optimizedCurrentTokenSource3(source, from); + case 4: + return optimizedCurrentTokenSource4(source, from); + case 5: + return optimizedCurrentTokenSource5(source, from); + case 6: + return optimizedCurrentTokenSource6(source, from); + case 0: + return CHAR_ARRAY0; + } + return Arrays.copyOfRange(source, from, to); + } + + private final char[] optimizedCurrentTokenSource2(char[] source, int startPosition) { + + char[] src = source; + int start = startPosition; + char c0, c1; + int hash = (((c0 = src[start]) << 6) + (c1 = src[start + 1])) % TABLE_SIZE; + char[][] table = this.charArray_length[0][hash]; + int i = this.newEntry2; + while (++i < INTERNAL_TABLE_SIZE) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1])) + return charArray; + } + // ---------other side--------- + i = -1; + int max = this.newEntry2; + while (++i <= max) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1])) + return charArray; + } + // --------add the entry------- + if (++max >= INTERNAL_TABLE_SIZE) + max = 0; + char[] r; + System.arraycopy(src, start, r = new char[2], 0, 2); + return table[this.newEntry2 = max] = r; + } + + private final char[] optimizedCurrentTokenSource3(char[] source, int startPosition) { + char[] src = source; + int start = startPosition; + char c0, c1 = src[start + 1], c2; + int hash = (((c0 = src[start]) << 6) + (c2 = src[start + 2])) % TABLE_SIZE; + char[][] table = this.charArray_length[1][hash]; + int i = this.newEntry3; + while (++i < INTERNAL_TABLE_SIZE) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) + return charArray; + } + // ---------other side--------- + i = -1; + int max = this.newEntry3; + while (++i <= max) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) + return charArray; + } + // --------add the entry------- + if (++max >= INTERNAL_TABLE_SIZE) + max = 0; + char[] r; + System.arraycopy(src, start, r = new char[3], 0, 3); + return table[this.newEntry3 = max] = r; + } + + private final char[] optimizedCurrentTokenSource4(char[] source, int startPosition) { + char[] src = source; + int start = startPosition; + char c0, c1 = src[start + 1], c2, c3 = src[start + 3]; + int hash = (((c0 = src[start]) << 6) + (c2 = src[start + 2])) % TABLE_SIZE; + char[][] table = this.charArray_length[2][hash]; + int i = this.newEntry4; + while (++i < INTERNAL_TABLE_SIZE) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3])) + return charArray; + } + // ---------other side--------- + i = -1; + int max = this.newEntry4; + while (++i <= max) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3])) + return charArray; + } + // --------add the entry------- + if (++max >= INTERNAL_TABLE_SIZE) + max = 0; + char[] r; + System.arraycopy(src, start, r = new char[4], 0, 4); + return table[this.newEntry4 = max] = r; + } + + private final char[] optimizedCurrentTokenSource5(char[] source, int startPosition) { + char[] src = source; + int start = startPosition; + char c0, c1 = src[start + 1], c2, c3 = src[start + 3], c4; + int hash = (((c0 = src[start]) << 12) + ((c2 = src[start + 2]) << 6) + (c4 = src[start + 4])) % TABLE_SIZE; + char[][] table = this.charArray_length[3][hash]; + int i = this.newEntry5; + while (++i < INTERNAL_TABLE_SIZE) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) + && (c4 == charArray[4])) + return charArray; + } + // ---------other side--------- + i = -1; + int max = this.newEntry5; + while (++i <= max) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) + && (c4 == charArray[4])) + return charArray; + } + // --------add the entry------- + if (++max >= INTERNAL_TABLE_SIZE) + max = 0; + char[] r; + System.arraycopy(src, start, r = new char[5], 0, 5); + return table[this.newEntry5 = max] = r; + } + + private final char[] optimizedCurrentTokenSource6(char[] source, int startPosition) { + char[] src = source; + int start = startPosition; + char c0, c1 = src[start + 1], c2, c3 = src[start + 3], c4, c5 = src[start + 5]; + int hash = (((c0 = src[start]) << 12) + ((c2 = src[start + 2]) << 6) + (c4 = src[start + 4])) % TABLE_SIZE; + char[][] table = this.charArray_length[4][hash]; + int i = this.newEntry6; + while (++i < INTERNAL_TABLE_SIZE) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) + && (c4 == charArray[4]) && (c5 == charArray[5])) + return charArray; + } + // ---------other side--------- + i = -1; + int max = this.newEntry6; + while (++i <= max) { + char[] charArray = table[i]; + if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) + && (c4 == charArray[4]) && (c5 == charArray[5])) + return charArray; + } + // --------add the entry------- + if (++max >= INTERNAL_TABLE_SIZE) + max = 0; + char[] r; + System.arraycopy(src, start, r = new char[6], 0, 6); + return table[this.newEntry6 = max] = r; + } +} diff --git a/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/util/PublicScanner.java b/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/util/PublicScanner.java index 398cfab7af..72b57e5c28 100644 --- a/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/util/PublicScanner.java +++ b/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/util/PublicScanner.java @@ -22,6 +22,7 @@ import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants; import org.eclipse.jdt.internal.compiler.parser.NLSTag; import org.eclipse.jdt.internal.compiler.parser.ScannerHelper; import org.eclipse.jdt.internal.compiler.parser.TerminalTokens; +import org.eclipse.jdt.internal.compiler.util.CharDeduplication; import org.eclipse.jdt.internal.compiler.util.Util; public class PublicScanner implements IScanner, ITerminalSymbols { @@ -127,41 +128,6 @@ public class PublicScanner implements IScanner, ITerminalSymbols { public static final String INVALID_UNDERSCORE = "Invalid_Underscore"; //$NON-NLS-1$ public static final String UNDERSCORES_IN_LITERALS_NOT_BELOW_17 = "Underscores_In_Literals_Not_Below_17"; //$NON-NLS-1$` - //----------------optimized identifier managment------------------ - static final char[] charArray_a = new char[] {'a'}, - charArray_b = new char[] {'b'}, - charArray_c = new char[] {'c'}, - charArray_d = new char[] {'d'}, - charArray_e = new char[] {'e'}, - charArray_f = new char[] {'f'}, - charArray_g = new char[] {'g'}, - charArray_h = new char[] {'h'}, - charArray_i = new char[] {'i'}, - charArray_j = new char[] {'j'}, - charArray_k = new char[] {'k'}, - charArray_l = new char[] {'l'}, - charArray_m = new char[] {'m'}, - charArray_n = new char[] {'n'}, - charArray_o = new char[] {'o'}, - charArray_p = new char[] {'p'}, - charArray_q = new char[] {'q'}, - charArray_r = new char[] {'r'}, - charArray_s = new char[] {'s'}, - charArray_t = new char[] {'t'}, - charArray_u = new char[] {'u'}, - charArray_v = new char[] {'v'}, - charArray_w = new char[] {'w'}, - charArray_x = new char[] {'x'}, - charArray_y = new char[] {'y'}, - charArray_z = new char[] {'z'}; - - static final char[] initCharArray = - new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'}; - static final int TableSize = 30, InternalTableSize = 6; //30*6 =210 entries - - public static final int OptimizedLength = 6; - public /*static*/ final char[][][][] charArray_length = - new char[OptimizedLength - 1][TableSize][InternalTableSize][]; // support for detecting non-externalized string literals public static final char[] TAG_PREFIX= "//$NON-NLS-".toCharArray(); //$NON-NLS-1$ public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length; @@ -176,20 +142,6 @@ public class PublicScanner implements IScanner, ITerminalSymbols { // generic support public boolean returnOnlyGreater = false; - /*static*/ { - for (int i = 0; i < OptimizedLength - 1; i++) { - for (int j = 0; j < TableSize; j++) { - for (int k = 0; k < InternalTableSize; k++) { - this.charArray_length[i][j][k] = initCharArray; - } - } - } - } - /*static*/ int newEntry2 = 0, - newEntry3 = 0, - newEntry4 = 0, - newEntry5 = 0, - newEntry6 = 0; public boolean insideRecovery = false; public static final int RoundBracket = 0; @@ -206,6 +158,8 @@ public class PublicScanner implements IScanner, ITerminalSymbols { // text block support - 13 /* package */ int rawStart = -1; + private CharDeduplication deduplication = CharDeduplication.getThreadLocalInstance(); + public PublicScanner() { this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/); } @@ -456,24 +410,9 @@ public char[] getCurrentIdentifierSource() { } int length = this.currentPosition - this.startPosition; if (length == this.eofPosition) return this.source; - switch (length) { // see OptimizedLength - case 1 : - return optimizedCurrentTokenSource1(); - case 2 : - return optimizedCurrentTokenSource2(); - case 3 : - return optimizedCurrentTokenSource3(); - case 4 : - return optimizedCurrentTokenSource4(); - case 5 : - return optimizedCurrentTokenSource5(); - case 6 : - return optimizedCurrentTokenSource6(); - } - char[] result = new char[length]; - System.arraycopy(this.source, this.startPosition, result, 0, length); - return result; + return this.deduplication.sharedCopyOfRange(this.source, this.startPosition, this.currentPosition); } + @Override public int getCurrentTokenEndPosition(){ return this.currentPosition - 1; @@ -2387,242 +2326,6 @@ public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException { return CharOperation.isWhitespace(this.currentCharacter); } -final char[] optimizedCurrentTokenSource1() { - //return always the same char[] build only once - - //optimization at no speed cost of 99.5 % of the singleCharIdentifier - char charOne = this.source[this.startPosition]; - switch (charOne) { - case 'a' : - return charArray_a; - case 'b' : - return charArray_b; - case 'c' : - return charArray_c; - case 'd' : - return charArray_d; - case 'e' : - return charArray_e; - case 'f' : - return charArray_f; - case 'g' : - return charArray_g; - case 'h' : - return charArray_h; - case 'i' : - return charArray_i; - case 'j' : - return charArray_j; - case 'k' : - return charArray_k; - case 'l' : - return charArray_l; - case 'm' : - return charArray_m; - case 'n' : - return charArray_n; - case 'o' : - return charArray_o; - case 'p' : - return charArray_p; - case 'q' : - return charArray_q; - case 'r' : - return charArray_r; - case 's' : - return charArray_s; - case 't' : - return charArray_t; - case 'u' : - return charArray_u; - case 'v' : - return charArray_v; - case 'w' : - return charArray_w; - case 'x' : - return charArray_x; - case 'y' : - return charArray_y; - case 'z' : - return charArray_z; - default : - return new char[] {charOne}; - } -} -final char[] optimizedCurrentTokenSource2() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0 , c1; - int hash = (((c0=src[start]) << 6) + (c1=src[start+1])) % TableSize; - char[][] table = this.charArray_length[0][hash]; - int i = this.newEntry2; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry2; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[2], 0, 2); - //newIdentCount++; - return table[this.newEntry2 = max] = r; //(r = new char[] {c0, c1}); -} -final char[] optimizedCurrentTokenSource3() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1=src[start+1], c2; - int hash = (((c0=src[start])<< 6) + (c2=src[start+2])) % TableSize; -// int hash = ((c0 << 12) + (c1<< 6) + c2) % TableSize; - char[][] table = this.charArray_length[1][hash]; - int i = this.newEntry3; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry3; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[3], 0, 3); - //newIdentCount++; - return table[this.newEntry3 = max] = r; //(r = new char[] {c0, c1, c2}); -} -final char[] optimizedCurrentTokenSource4() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1 = src[start+1], c2, c3 = src[start+3]; - int hash = (((c0=src[start]) << 6) + (c2=src[start+2])) % TableSize; -// int hash = (int) (((((long) c0) << 18) + (c1 << 12) + (c2 << 6) + c3) % TableSize); - char[][] table = this.charArray_length[2][hash]; - int i = this.newEntry4; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry4; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[4], 0, 4); - //newIdentCount++; - return table[this.newEntry4 = max] = r; //(r = new char[] {c0, c1, c2, c3}); -} -final char[] optimizedCurrentTokenSource5() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1 = src[start+1], c2, c3 = src[start+3], c4; - int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize; -// int hash = (int) (((((long) c0) << 24) + (((long) c1) << 18) + (c2 << 12) + (c3 << 6) + c4) % TableSize); - char[][] table = this.charArray_length[3][hash]; - int i = this.newEntry5; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry5; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[5], 0, 5); - //newIdentCount++; - return table[this.newEntry5 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4}); -} -final char[] optimizedCurrentTokenSource6() { - //try to return the same char[] build only once - - char[] src = this.source; - int start = this.startPosition; - char c0, c1 = src[start+1], c2, c3 = src[start+3], c4, c5 = src[start+5]; - int hash = (((c0=src[start]) << 12) +((c2=src[start+2]) << 6) + (c4=src[start+4])) % TableSize; -// int hash = (int)(((((long) c0) << 32) + (((long) c1) << 24) + (((long) c2) << 18) + (c3 << 12) + (c4 << 6) + c5) % TableSize); - char[][] table = this.charArray_length[4][hash]; - int i = this.newEntry6; - while (++i < InternalTableSize) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4]) - && (c5 == charArray[5])) - return charArray; - } - //---------other side--------- - i = -1; - int max = this.newEntry6; - while (++i <= max) { - char[] charArray = table[i]; - if ((c0 == charArray[0]) - && (c1 == charArray[1]) - && (c2 == charArray[2]) - && (c3 == charArray[3]) - && (c4 == charArray[4]) - && (c5 == charArray[5])) - return charArray; - } - //--------add the entry------- - if (++max >= InternalTableSize) max = 0; - char[] r; - System.arraycopy(src, start, r= new char[6], 0, 6); - //newIdentCount++; - return table[this.newEntry6 = max] = r; //(r = new char[] {c0, c1, c2, c3, c4, c5}); -} - private void parseTags() { int position = 0; final int currentStartPosition = this.startPosition; |