diff options
author | Matthew Khouzam | 2012-07-19 17:54:46 +0000 |
---|---|---|
committer | Matthew Khouzam | 2012-07-19 23:21:35 +0000 |
commit | ace12772c67c7b9a04ed2c11ac1f8a07bb577218 (patch) | |
tree | 17b28c467097c7aee7cf31320bc31b9473ce3629 | |
parent | cc5f71b59aefaed909c7f4b8c9e6ae9ebd46afa5 (diff) | |
download | org.eclipse.linuxtools-ace12772c67c7b9a04ed2c11ac1f8a07bb577218.tar.gz org.eclipse.linuxtools-ace12772c67c7b9a04ed2c11ac1f8a07bb577218.tar.xz org.eclipse.linuxtools-ace12772c67c7b9a04ed2c11ac1f8a07bb577218.zip |
Fix trace and experiment seeking performance bug
Change-Id: I6c59ac7ba52d80ab4bab597da93ea7adb950158d
Signed-off-by: Matthew Khouzam <matthew.khouzam@ericsson.com>
Reviewed-on: https://git.eclipse.org/r/6817
Reviewed-by: Alexandre Montplaisir <alexmonthy@voxpopuli.im>
IP-Clean: Alexandre Montplaisir <alexmonthy@voxpopuli.im>
Tested-by: Alexandre Montplaisir <alexmonthy@voxpopuli.im>
5 files changed, 381 insertions, 101 deletions
diff --git a/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/CtfTmfLightweightContextTest.java b/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/CtfTmfLightweightContextTest.java new file mode 100644 index 0000000000..6dfaa9df18 --- /dev/null +++ b/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/CtfTmfLightweightContextTest.java @@ -0,0 +1,121 @@ +/******************************************************************************* + * Copyright (c) 2012 Ericsson + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v1.0 which + * accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Matthew Khouzam - Initial implementation + * Alexandre Montplaisir + *******************************************************************************/ + +package org.eclipse.linuxtools.tmf.core.tests.ctfadaptor; + +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; + +import org.eclipse.core.resources.IResource; +import org.eclipse.linuxtools.tmf.core.ctfadaptor.CtfTmfEvent; +import org.eclipse.linuxtools.tmf.core.ctfadaptor.CtfTmfLightweightContext; +import org.eclipse.linuxtools.tmf.core.ctfadaptor.CtfTmfTrace; +import org.eclipse.linuxtools.tmf.core.exceptions.TmfTraceException; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests for the CtfTmfLightweightContext class + * + * @author Matthew Khouzam + * @version 1.1 + */ +public class CtfTmfLightweightContextTest { + + private static final String PATH = TestParams.getPath(); + private static final long begin = 1332170682440133097L; /* Trace start time */ + private static final long end = 1332170692664579801L; /* Trace end time */ + + private CtfTmfTrace fixture; + + private class SeekerThread extends Thread { + long val; + + public void setVal(long val) { + this.val = val; + } + } + + /** + * Pre-test initialization + * + * @throws TmfTraceException + * If the trace couldn't be init'ed, which shouldn't happen. + */ + @Before + public void setUp() throws TmfTraceException { + fixture = new CtfTmfTrace(); + fixture.initTrace((IResource) null, PATH, CtfTmfEvent.class); + } + + /** + * Index all the events in the test trace. + */ + @Test + public void testIndexing() { + CtfTmfLightweightContext context = new CtfTmfLightweightContext(fixture); + context.seek(0); + + int count = 0; + while (fixture.getNext(context) != null) { + count++; + } + assertTrue(count > 0); + } + + /** + * Context fuzzer. Use an amount of contexts greater than the size of the + * iterator cache and have them access the trace in parallel. + * + * @throws TmfTraceException + * Would fail the test + * @throws InterruptedException + * Would fail the test + */ + @Test + public void testTooManyContexts() throws TmfTraceException, InterruptedException { + final int lwcCount = 101; + double increment = (end - begin) / lwcCount; + final ArrayList<Long> vals = new ArrayList<Long>(); + final ArrayList<Thread> threads = new ArrayList<Thread>(); + final ArrayList<CtfTmfLightweightContext> tooManyContexts = new ArrayList<CtfTmfLightweightContext>(); + + for (double i = begin; i < end; i += increment) { + SeekerThread thread = new SeekerThread() { + @Override + public void run() { + CtfTmfLightweightContext lwc = new CtfTmfLightweightContext(fixture); + lwc.seek(val); + fixture.getNext(lwc); + synchronized(fixture){ + vals.add(lwc.getCurrentEvent().getTimestampValue()); + tooManyContexts.add(lwc); + } + } + }; + thread.setVal((long)i); + threads.add(thread); + thread.start(); + } + + for( Thread t: threads){ + t.join(); + } + + for( Long val : vals){ + assertTrue(val >= begin); + assertTrue(val <= end); + } + } +} diff --git a/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/TestAll.java b/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/TestAll.java index cc8e4b3d75..92d283f849 100644 --- a/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/TestAll.java +++ b/lttng/org.eclipse.linuxtools.tmf.core.tests/src/org/eclipse/linuxtools/tmf/core/tests/ctfadaptor/TestAll.java @@ -34,7 +34,8 @@ import org.junit.runners.Suite; CtfTmfEventTest.class, CtfTmfEventTypeTest.class, CtfTmfTimestampTest.class, - CtfTmfTraceTest.class + CtfTmfTraceTest.class, + CtfTmfLightweightContextTest.class }) public class TestAll { diff --git a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfIteratorManager.java b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfIteratorManager.java new file mode 100644 index 0000000000..865c551544 --- /dev/null +++ b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfIteratorManager.java @@ -0,0 +1,189 @@ +/******************************************************************************* + * Copyright (c) 2012 Ericsson + * + * All rights reserved. This program and the accompanying materials are made + * available under the terms of the Eclipse Public License v1.0 which + * accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: Matthew Khouzam - Initial API and implementation + *******************************************************************************/ +package org.eclipse.linuxtools.tmf.core.ctfadaptor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Random; + +/** + * Ctf Iterator Manager, allows mapping of iterators (a limited resource) to + * contexts (many many resources). + * + * @author Matthew Khouzam + * @version 1.0 + * @since 1.1 + */ +public abstract class CtfIteratorManager { + /* + * A side note synchronized works on the whole object, Therefore add and + * remove will be thread safe. + */ + + /* + * The map of traces to trace managers. + */ + private static HashMap<CtfTmfTrace, CtfTraceManager> map = new HashMap<CtfTmfTrace, CtfTraceManager>(); + + /** + * Registers a trace to the iterator manager, the trace can now get + * iterators. + * + * @param trace + * the trace to register. + */ + public static synchronized void addTrace(final CtfTmfTrace trace) { + map.put(trace, new CtfTraceManager(trace)); + } + + /** + * Removes a trace to the iterator manager. + * + * @param trace + * the trace to register. + */ + public static synchronized void removeTrace(final CtfTmfTrace trace) { + map.remove(trace); + } + + /** + * Get an iterator for a given trace and context. + * + * @param trace + * the trace + * @param ctx + * the context + * @return the iterator + */ + public static synchronized CtfIterator getIterator(final CtfTmfTrace trace, + final CtfTmfLightweightContext ctx) { + return map.get(trace).getIterator(ctx); + } +} + +/** + * A trace manager + * + * @author Matthew Khouzam + */ +class CtfTraceManager { + /* + * Cache size. Under 1023 on linux32 systems. Number of file handles + * created. + */ + private final static int MAX_SIZE = 100; + /* + * The map of the cache. + */ + private final HashMap<CtfTmfLightweightContext, CtfIterator> fMap; + /* + * An array pointing to the same cache. this allows fast "random" accesses. + */ + private final ArrayList<CtfTmfLightweightContext> fRandomAccess; + /* + * The parent trace + */ + private final CtfTmfTrace fTrace; + /* + * Random number generator + */ + private final Random fRnd; + + public CtfTraceManager(CtfTmfTrace trace) { + fMap = new HashMap<CtfTmfLightweightContext, CtfIterator>(); + fRandomAccess = new ArrayList<CtfTmfLightweightContext>(); + fRnd = new Random(System.nanoTime()); + fTrace = trace; + } + + /** + * This needs explaining: the iterator table is effectively a cache. + * Originally the contexts had a 1 to 1 structure with the file handles of a + * trace. This failed since there is a limit to how many file handles we can + * have opened simultaneously. Then a round-robin scheme was implemented, + * this lead up to a two competing contexts syncing up and using the same + * file handler, causing horrible slowdowns. Now a random replacement + * algorithm is selected. This is the same as used by arm processors, and it + * works quite well when many cores so this looks promising for very + * multi-threaded systems. + * + * @param context + * the context to look up + * @return the iterator refering to the context + */ + public CtfIterator getIterator(final CtfTmfLightweightContext context) { + /* + * if the element is in the map, we don't need to do anything else. + */ + CtfIterator retVal = fMap.get(context); + if (retVal == null) { + /* + * Assign an iterator to a context, this means we will need to seek + * at the end. + */ + if (fRandomAccess.size() < MAX_SIZE) { + /* + * if we're not full yet, just add an element. + */ + retVal = new CtfIterator(fTrace); + addElement(context, retVal); + + } else { + /* + * if we're full, randomly replace an element + */ + retVal = replaceRandomElement(context); + } + retVal.seek((Long) context.getLocation().getLocation()); + } + return retVal; + } + + /** + * Add a pair of context and element to the hashmap and the arraylist. + * + * @param context + * the context + * @param elem + * the iterator + */ + private void addElement(final CtfTmfLightweightContext context, + final CtfIterator elem) { + fMap.put(context, elem); + fRandomAccess.add(context); + } + + /** + * Replace a random element + * + * @param context + * the context to swap in + * @return the iterator of the removed elements. + */ + private CtfIterator replaceRandomElement( + final CtfTmfLightweightContext context) { + /* + * This needs some explanation too: We need to select a random victim + * and remove it. The order of the elements is not important, so instead + * of just calling arraylist.remove(element) which has an O(n) + * complexity, we pick an random number. The element is swapped out of + * the array and removed and replaced in the hashmap. + */ + final int size = fRandomAccess.size(); + final int pos = fRnd.nextInt(size); + final CtfTmfLightweightContext victim = fRandomAccess.get(pos); + fRandomAccess.set(pos, context); + final CtfIterator elem = fMap.remove(victim); + fMap.put(context, elem); + return elem; + } + +} diff --git a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLightweightContext.java b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLightweightContext.java index 98da0bf7e7..ef0b7e7060 100644 --- a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLightweightContext.java +++ b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLightweightContext.java @@ -27,38 +27,40 @@ import org.eclipse.linuxtools.tmf.core.trace.ITmfLocation; public class CtfTmfLightweightContext implements ITmfContext { // ------------------------------------------- - // Constants - // ------------------------------------------- - private static final int MAX_COLLISIONS = 10; - - // ------------------------------------------- // Fields // ------------------------------------------- private CtfLocation curLocation; private long curRank; - private int collisions; - private CtfIterator fSeeker; - final private ArrayList<CtfIterator> fIteratorPool; - private ListIterator<CtfIterator> fCurrentIterator; + private final CtfTmfTrace fTrace; // ------------------------------------------- // Constructor // ------------------------------------------- /** + * Deprecated, use CtfTmfLightweightContext( CtfTmfTrace please ) * * @param iters * the shared iterator pool. * @param pos * the iterator position. */ + @Deprecated public CtfTmfLightweightContext(ArrayList<CtfIterator> iters, ListIterator<CtfIterator> pos) { - fIteratorPool = iters; - fCurrentIterator = pos; - fSeeker = getIterator(); - curLocation = new CtfLocation((Long)null); - collisions = 0; + fTrace = iters.get(0).getCtfTmfTrace(); + curLocation = new CtfLocation((Long) null); + } + + /** + * + * @param ctfTmfTrace + * the parent trace + * @since 1.1 + */ + public CtfTmfLightweightContext(CtfTmfTrace ctfTmfTrace) { + fTrace = ctfTmfTrace; + curLocation = new CtfLocation((Long) null); } // ------------------------------------------- @@ -83,7 +85,7 @@ public class CtfTmfLightweightContext implements ITmfContext { @Override public void setLocation(ITmfLocation<? extends Comparable<?>> location) { curLocation = (CtfLocation) location; - updateLocation(); + getIterator().seek(curLocation.getLocation()); } @Override @@ -105,21 +107,21 @@ public class CtfTmfLightweightContext implements ITmfContext { /** * Gets the current event. Wrapper to help CtfTmfTrace + * * @return The event or null */ public synchronized CtfTmfEvent getCurrentEvent() { - updateLocation(); - return fSeeker.getCurrentEvent(); + return getIterator().getCurrentEvent(); } /** * Advances to a the next event. Wrapper to help CtfTmfTrace + * * @return success or not */ public synchronized boolean advance() { - updateLocation(); - boolean retVal = fSeeker.advance(); - CtfTmfEvent currentEvent = fSeeker.getCurrentEvent(); + boolean retVal = getIterator().advance(); + CtfTmfEvent currentEvent = getIterator().getCurrentEvent(); if (currentEvent != null) { curLocation.setLocation(currentEvent.getTimestampValue()); } else { @@ -136,14 +138,14 @@ public class CtfTmfLightweightContext implements ITmfContext { /** * Seeks to a given timestamp. Wrapper to help CtfTmfTrace - * @param timestamp desired timestamp + * + * @param timestamp + * desired timestamp * @return success or not */ public synchronized boolean seek(final long timestamp) { curLocation.setLocation(timestamp); - collisions = 0; - fSeeker = getIterator(); - return updateLocation(); + return getIterator().seek(timestamp); } /* @@ -153,8 +155,7 @@ public class CtfTmfLightweightContext implements ITmfContext { */ @Override public CtfTmfLightweightContext clone() { - CtfTmfLightweightContext ret = new CtfTmfLightweightContext( - fIteratorPool, fCurrentIterator); + CtfTmfLightweightContext ret = new CtfTmfLightweightContext(fTrace); ret.curLocation = curLocation.clone(); ret.curRank = curRank; return ret; @@ -164,40 +165,12 @@ public class CtfTmfLightweightContext implements ITmfContext { // Private helpers // ------------------------------------------- /** - * This updates the position of an iterator to the location(curLocation) - * Since the iterators are in a pool to not exhaust the number of file - * pointers some of them can be shared. This means there can be collisions - * between contexts fighting over the same resource. A heuristic is applied - * that if there are MAX_COLLISIONS collisions in a row, the iterator is - * changed for the next one in the iterator pool. + * Get iterator, called every time to get an iterator, no local copy is + * stored so that there is no need to "update" * - * @return true if the location is correct. - */ - private synchronized boolean updateLocation() { - if (!curLocation.getLocation().equals( - (fSeeker.getLocation().getLocation()))) { - collisions++; - if (collisions > MAX_COLLISIONS) { - fSeeker = getIterator(); - collisions = 0; - } - fSeeker.setRank(curRank); - return fSeeker.seek(curLocation.getLocation()); - } - collisions = 0; - return true; - } - - /** - * gets the next iterator in a pool. - * - * @return + * @return an iterator */ private CtfIterator getIterator() { - if (!fCurrentIterator.hasNext()) { - fCurrentIterator = fIteratorPool.listIterator(0); - } - return fCurrentIterator.next(); + return CtfIteratorManager.getIterator(fTrace, this); } - } diff --git a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfTrace.java b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfTrace.java index ff9eb25ee3..1fd7524908 100644 --- a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfTrace.java +++ b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfTrace.java @@ -11,9 +11,6 @@ package org.eclipse.linuxtools.tmf.core.ctfadaptor; -import java.util.ArrayList; -import java.util.ListIterator; - import org.eclipse.core.resources.IProject; import org.eclipse.core.resources.IResource; import org.eclipse.core.runtime.CoreException; @@ -45,7 +42,6 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse * Default cache size for CTF traces */ protected static final int DEFAULT_CACHE_SIZE = 50000; - private static final int ITER_POOL_SIZE = 128; //------------------------------------------- // Fields @@ -57,15 +53,7 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse /* Reference to the CTF Trace */ private CTFTrace fTrace; - /* - * The iterator pool. This is a necessary change since larger traces will - * need many contexts and each context must have to a file pointer. Since - * the OS supports only so many handles on a given file, but the UI must - * still be responsive, parallel seeks (up to ITER_POOL_SIZE requests) - * can be made with a fast response time. - * */ - private ArrayList<CtfIterator> fIterators; - private ListIterator<CtfIterator> nextIter; + //------------------------------------------- // TmfTrace Overrides @@ -92,19 +80,17 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse try { this.fTrace = new CTFTrace(path); - fIterators = new ArrayList<CtfIterator>(ITER_POOL_SIZE); - for(int i = 0 ; i < ITER_POOL_SIZE; i++){ - fIterators.add(new CtfIterator(this, 0, 0)); - } - nextIter = fIterators.listIterator(0); + CtfIteratorManager.addTrace(this); + CtfTmfLightweightContext ctx; /* Set the start and (current) end times for this trace */ - final CtfIterator iterator = getIterator(); - if(iterator.getLocation().equals(CtfIterator.NULL_LOCATION)) { + ctx = new CtfTmfLightweightContext(this); + ctx.setLocation(new CtfLocation(0L)); + if(ctx.getLocation().equals(CtfIterator.NULL_LOCATION)) { /* Handle the case where the trace is empty */ this.setStartTime(TmfTimestamp.BIG_BANG); } else { - this.setStartTime(iterator.getCurrentEvent().getTimestamp()); - this.setEndTime(iterator.getCurrentEvent().getTimestamp()); + this.setStartTime(ctx.getCurrentEvent().getTimestamp()); + this.setEndTime(ctx.getCurrentEvent().getTimestamp()); } } catch (final CTFReaderException e) { @@ -129,6 +115,15 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse } } + /* (non-Javadoc) + * @see org.eclipse.linuxtools.tmf.core.trace.TmfTrace#dispose() + */ + @Override + public synchronized void dispose() { + CtfIteratorManager.removeTrace(this); + super.dispose(); + } + /** * Method validate. * @param project IProject @@ -162,25 +157,19 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse @Override public double getLocationRatio(ITmfLocation<?> location) { final CtfLocation curLocation = (CtfLocation) location; - CtfIterator iterator = getIterator(); - CtfTmfLightweightContext ctx = new CtfTmfLightweightContext(fIterators, nextIter); - ctx.setLocation(curLocation); - ctx.seek(curLocation.getLocation()); - long currentTime = ((Long)ctx.getLocation().getLocation()); - - return ((double) currentTime - iterator.getStartTime()) - / (iterator.getEndTime() - iterator.getStartTime()); + final CtfTmfLightweightContext context = new CtfTmfLightweightContext(this); + context.setLocation(curLocation); + context.seek(curLocation.getLocation()); + final long currentTime = ((Long)context.getLocation().getLocation()); + final long startTime = getIterator(this, context).getStartTime(); + final long endTime = getIterator(this, context).getEndTime(); + return ((double) currentTime - startTime) + / (endTime - startTime); } - /** - * @return - */ - private CtfIterator getIterator() { - if( !nextIter.hasNext()){ - nextIter = fIterators.listIterator(0); - } - return nextIter.next(); - } + + + /* (non-Javadoc) * @see org.eclipse.linuxtools.tmf.core.trace.TmfTrace#seekEvent(org.eclipse.linuxtools.tmf.core.event.ITmfTimestamp) @@ -188,7 +177,7 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse @Override public synchronized ITmfContext seekEvent(ITmfTimestamp timestamp) { if( timestamp instanceof CtfTmfTimestamp){ - CtfTmfLightweightContext iter = new CtfTmfLightweightContext(fIterators, nextIter); + CtfTmfLightweightContext iter = new CtfTmfLightweightContext(this); iter.seek(timestamp.getValue()); return iter; } @@ -203,7 +192,7 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse @Override public ITmfContext seekEvent(final ITmfLocation<?> location) { CtfLocation currentLocation = (CtfLocation) location; - CtfTmfLightweightContext context = new CtfTmfLightweightContext(fIterators, nextIter); + CtfTmfLightweightContext context = new CtfTmfLightweightContext(this); /* * The rank is set to 0 if the iterator seeks the beginning. If not, it * will be set to UNKNOWN_RANK, since CTF traces don't support seeking @@ -227,7 +216,7 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse @Override public ITmfContext seekEvent(double ratio) { - CtfTmfLightweightContext context = new CtfTmfLightweightContext(fIterators, nextIter); + CtfTmfLightweightContext context = new CtfTmfLightweightContext(this); context.seek((long) (this.getNbEvents() * ratio)); context.setRank(ITmfContext.UNKNOWN_RANK); return context; @@ -359,4 +348,11 @@ public class CtfTmfTrace extends TmfTrace<CtfTmfEvent> implements ITmfEventParse setCacheSize(DEFAULT_CACHE_SIZE); } + //------------------------------------------- + // Helpers + //------------------------------------------- + + private static CtfIterator getIterator(CtfTmfTrace trace, CtfTmfLightweightContext context) { + return CtfIteratorManager.getIterator(trace, context); + } } |