ctf: Use "meta events" for lost events (bug 408373)

Instead of sending one event object for every single event reported as
lost in a CTF trace, only send one CTFEvent indicating how many events
were lost, in which time range.

On the TMF (ctfadaptor) side, we add a new CtfTmfLostEvent class to
wrap those. This way, only one entry shows up in the Events table,
indicating how many events were lost at that point in the trace.

Change-Id: I0c70ec2d5fc77134b67426d053a4d2353e710ab8
Signed-off-by: Simon Delisle <simon.delisle@ericsson.com>
Signed-off-by: Jean-Christian Kouamé <kadjo.gwandy.jean-christian.kouame@ericsson.com>
Signed-off-by: Alexandre Montplaisir <alexmonthy@voxpopuli.im>
Reviewed-on: https://git.eclipse.org/r/15491
IP-Clean: Matthew Khouzam <matthew.khouzam@ericsson.com>
Tested-by: Matthew Khouzam <matthew.khouzam@ericsson.com>
Reviewed-by: Matthew Khouzam <matthew.khouzam@ericsson.com>

7 files changed, 208 insertions, 24 deletions

diff --git a/lttng/org.eclipse.linuxtools.ctf.core/META-INF/MANIFEST.MF b/lttng/org.eclipse.linuxtools.ctf.core/META-INF/MANIFEST.MF
index e76423679f..ac876444c0 100644
--- a/lttng/org.eclipse.linuxtools.ctf.core/META-INF/MANIFEST.MF
+++ b/lttng/org.eclipse.linuxtools.ctf.core/META-INF/MANIFEST.MF
@@ -10,7 +10,8 @@ Bundle-ActivationPolicy: lazy
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Require-Bundle: org.eclipse.core.runtime,
  org.eclipse.linuxtools.ctf.parser;bundle-version="1.0.0"
-Export-Package: org.eclipse.linuxtools.ctf.core.event,
+Export-Package: org.eclipse.linuxtools.ctf.core,
+ org.eclipse.linuxtools.ctf.core.event,
  org.eclipse.linuxtools.ctf.core.event.io,
  org.eclipse.linuxtools.ctf.core.event.types,
  org.eclipse.linuxtools.ctf.core.trace,
diff --git a/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/CTFStrings.java b/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/CTFStrings.java
new file mode 100644
index 0000000000..e942afa8e7
--- /dev/null
+++ b/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/CTFStrings.java
@@ -0,0 +1,38 @@
+/*******************************************************************************
+ * Copyright (c) 2013 Ericsson
+ *
+ * All rights reserved. This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License v1.0 which
+ * accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *   Alexandre Montplaisir - Initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.linuxtools.ctf.core;
+
+/**
+ * Non-externalized strings for use with the CTF plugin (event names, field
+ * names, etc.)
+ *
+ * @author Alexandre Montplaisir
+ * @since 2.1
+ */
+@SuppressWarnings("nls")
+public interface CTFStrings {
+
+    /** Event name for lost events */
+    static final String LOST_EVENT_NAME = "Lost event";
+
+    /**
+     * Name of the field in lost events indicating how many actual events were
+     * lost
+     */
+    static final String LOST_EVENTS_FIELD = "Lost events";
+
+    /**
+     * Name of the field in lost events indicating the time range
+     */
+    static final String LOST_EVENTS_DURATION = "duration";
+}
diff --git a/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/trace/StreamInputPacketReader.java b/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/trace/StreamInputPacketReader.java
index 678cb029a6..cfeca1abe9 100644
--- a/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/trace/StreamInputPacketReader.java
+++ b/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/ctf/core/trace/StreamInputPacketReader.java
@@ -16,6 +16,7 @@ import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel.MapMode;
 import java.util.Collection;
 
+import org.eclipse.linuxtools.ctf.core.CTFStrings;
 import org.eclipse.linuxtools.ctf.core.event.EventDefinition;
 import org.eclipse.linuxtools.ctf.core.event.IEventDeclaration;
 import org.eclipse.linuxtools.ctf.core.event.io.BitBuffer;
@@ -57,7 +58,7 @@ public class StreamInputPacketReader implements IDefinitionScope {
     /** Stream event header definition. */
     private final StructDefinition streamEventHeaderDef;
 
-    /** Stream event context definition.*/
+    /** Stream event context definition. */
     private final StructDefinition streamEventContextDef;
 
     /** Reference to the index entry of the current packet. */
@@ -74,11 +75,12 @@ public class StreamInputPacketReader implements IDefinitionScope {
     /** CPU id of current packet. */
     private int currentCpu = 0;
 
-    /** number of lost events in this packet */
-    private int lostSoFar;
-
     private int lostEventsInThisPacket;
 
+    private long lostEventsDuration;
+
+    private boolean hasLost = false;
+
     // ------------------------------------------------------------------------
     // Constructors
     // ------------------------------------------------------------------------
@@ -96,8 +98,6 @@ public class StreamInputPacketReader implements IDefinitionScope {
         bitBuffer = new BitBuffer();
         bitBuffer.setByteOrder(streamInputReader.getByteOrder());
 
-        lostSoFar = 0;
-
         /* Create trace packet header definition. */
         final Stream currentStream = streamInputReader.getStreamInput().getStream();
         StructDeclaration tracePacketHeaderDecl = currentStream.getTrace().getPacketHeader();
@@ -144,6 +144,7 @@ public class StreamInputPacketReader implements IDefinitionScope {
 
     /**
      * Dispose the StreamInputPacketReader
+     *
      * @since 2.0
      */
     public void dispose() {
@@ -206,6 +207,7 @@ public class StreamInputPacketReader implements IDefinitionScope {
      *            The index entry of the packet to switch to.
      */
     void setCurrentPacket(StreamInputPacketIndexEntry currentPacket) {
+        StreamInputPacketIndexEntry prevPacket = null;
         this.currentPacket = currentPacket;
 
         if (this.currentPacket != null) {
@@ -248,8 +250,23 @@ public class StreamInputPacketReader implements IDefinitionScope {
 
                 /* Read number of lost events */
                 lostEventsInThisPacket = (int) this.getCurrentPacket().getLostEvents();
-                lostSoFar = 0;
-
+                if (lostEventsInThisPacket != 0) {
+                    hasLost = true;
+                    /*
+                     * Compute the duration of the lost event time range. If the
+                     * current packet is the first packet, duration will be set
+                     * to 1.
+                     */
+                    long lostEventsStartTime;
+                    int index = this.streamInputReader.getStreamInput().getIndex().getEntries().indexOf(currentPacket);
+                    if (index == 0) {
+                        lostEventsStartTime = currentPacket.getTimestampBegin() + 1;
+                    } else {
+                        prevPacket = this.streamInputReader.getStreamInput().getIndex().getEntries().get(index - 1);
+                        lostEventsStartTime = prevPacket.getTimestampEnd();
+                    }
+                    lostEventsDuration = Math.abs(lostEventsStartTime - currentPacket.getTimestampBegin());
+                }
             }
 
             /*
@@ -271,7 +288,7 @@ public class StreamInputPacketReader implements IDefinitionScope {
      */
     public boolean hasMoreEvents() {
         if (currentPacket != null) {
-            return bitBuffer.position() < currentPacket.getContentSizeBits();
+            return hasLost || (bitBuffer.position() < currentPacket.getContentSizeBits());
         }
         return false;
     }
@@ -288,11 +305,12 @@ public class StreamInputPacketReader implements IDefinitionScope {
         /* Default values for those fields */
         long eventID = EventDeclaration.UNSET_EVENT_ID;
         long timestamp = 0;
-
-        if (lostEventsInThisPacket > lostSoFar) {
+        if (hasLost) {
+            hasLost = false;
             EventDefinition eventDef = EventDeclaration.getLostEventDeclaration().createDefinition(streamInputReader);
+            ((IntegerDefinition) eventDef.getFields().getDefinitions().get(CTFStrings.LOST_EVENTS_FIELD)).setValue(lostEventsInThisPacket);
+            ((IntegerDefinition) eventDef.getFields().getDefinitions().get(CTFStrings.LOST_EVENTS_DURATION)).setValue(lostEventsDuration);
             eventDef.setTimestamp(this.lastTimestamp);
-            ++lostSoFar;
             return eventDef;
         }
 
@@ -309,7 +327,10 @@ public class StreamInputPacketReader implements IDefinitionScope {
                 eventID = ((SimpleDatatypeDefinition) idDef).getIntegerValue();
             } // else, eventID remains 0
 
-            /* Get the timestamp from the event header (may be overridden later on) */
+            /*
+             * Get the timestamp from the event header (may be overridden later
+             * on)
+             */
             IntegerDefinition timestampDef = sehd.lookupInteger("timestamp"); //$NON-NLS-1$
             if (timestampDef != null) {
                 timestamp = calculateTimestamp(timestampDef);
@@ -331,7 +352,10 @@ public class StreamInputPacketReader implements IDefinitionScope {
                     eventID = ((IntegerDefinition) idIntegerDef).getValue();
                 }
 
-                /* Get the timestamp. This would overwrite any previous timestamp definition */
+                /*
+                 * Get the timestamp. This would overwrite any previous
+                 * timestamp definition
+                 */
                 Definition def = variantCurrentField.lookupDefinition("timestamp"); //$NON-NLS-1$
                 if (def instanceof IntegerDefinition) {
                     timestamp = calculateTimestamp((IntegerDefinition) def);
diff --git a/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/internal/ctf/core/event/EventDeclaration.java b/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/internal/ctf/core/event/EventDeclaration.java
index 28f5c7e06a..fa00efe121 100644
--- a/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/internal/ctf/core/event/EventDeclaration.java
+++ b/lttng/org.eclipse.linuxtools.ctf.core/src/org/eclipse/linuxtools/internal/ctf/core/event/EventDeclaration.java
@@ -12,12 +12,16 @@
 
 package org.eclipse.linuxtools.internal.ctf.core.event;
 
+import java.nio.ByteOrder;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
+import org.eclipse.linuxtools.ctf.core.CTFStrings;
 import org.eclipse.linuxtools.ctf.core.event.EventDefinition;
 import org.eclipse.linuxtools.ctf.core.event.IEventDeclaration;
+import org.eclipse.linuxtools.ctf.core.event.types.Encoding;
+import org.eclipse.linuxtools.ctf.core.event.types.IntegerDeclaration;
 import org.eclipse.linuxtools.ctf.core.event.types.StructDeclaration;
 import org.eclipse.linuxtools.ctf.core.trace.Stream;
 import org.eclipse.linuxtools.ctf.core.trace.StreamInputReader;
@@ -105,9 +109,15 @@ public class EventDeclaration implements IEventDeclaration {
      */
     public static synchronized EventDeclaration getLostEventDeclaration() {
         EventDeclaration lostEvent = new EventDeclaration();
+        IntegerDeclaration lostEventsDeclaration = new IntegerDeclaration(32, false, 10, ByteOrder.BIG_ENDIAN, Encoding.ASCII, null, 8);
+        IntegerDeclaration timestampDeclaration = new IntegerDeclaration(64, false, 10, ByteOrder.BIG_ENDIAN, Encoding.ASCII, null, 8);
+
         lostEvent.fields = new StructDeclaration(1);
+        lostEvent.fields.addField(CTFStrings.LOST_EVENTS_FIELD, lostEventsDeclaration);
+        lostEvent.fields.addField(CTFStrings.LOST_EVENTS_DURATION, timestampDeclaration);
         lostEvent.id = LOST_EVENT_ID;
-        lostEvent.name = "Lost event"; //$NON-NLS-1$
+        lostEvent.name = CTFStrings.LOST_EVENT_NAME;
+
         return lostEvent;
     }
 
diff --git a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEvent.java b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEvent.java
index ae002a0cc9..bc28e4ad91 100644
--- a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEvent.java
+++ b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEvent.java
@@ -35,7 +35,7 @@ import org.eclipse.linuxtools.tmf.core.trace.ITmfContext;
  * @author Alexandre Montplaisir
  * @since 2.0
  */
-public final class CtfTmfEvent extends TmfEvent
+public class CtfTmfEvent extends TmfEvent
         implements ITmfSourceLookup, ITmfModelLookup, ITmfCustomAttributes {
 
     // ------------------------------------------------------------------------
diff --git a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEventFactory.java b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEventFactory.java
index ead5231283..9178188050 100644
--- a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEventFactory.java
+++ b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfEventFactory.java
@@ -16,11 +16,15 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.eclipse.linuxtools.ctf.core.CTFStrings;
 import org.eclipse.linuxtools.ctf.core.event.EventDefinition;
+import org.eclipse.linuxtools.ctf.core.event.IEventDeclaration;
 import org.eclipse.linuxtools.ctf.core.event.types.Definition;
+import org.eclipse.linuxtools.ctf.core.event.types.IntegerDefinition;
 import org.eclipse.linuxtools.ctf.core.event.types.StructDefinition;
 import org.eclipse.linuxtools.tmf.core.event.ITmfEventField;
 import org.eclipse.linuxtools.tmf.core.event.TmfEventField;
+import org.eclipse.linuxtools.tmf.core.timestamp.TmfTimeRange;
 import org.eclipse.linuxtools.tmf.core.trace.ITmfContext;
 
 /**
@@ -54,9 +58,10 @@ public final class CtfTmfEventFactory {
             String fileName, CtfTmfTrace originTrace) {
 
         /* Prepare what to pass to CtfTmfEvent's constructor */
-        long ts = eventDef.getTimestamp();
-        CtfTmfTimestamp timestamp = new CtfTmfTimestamp(
-                    originTrace.getCTFTrace().timestampCyclesToNanos(ts));
+        final IEventDeclaration eventDecl = eventDef.getDeclaration();
+        final long ts = eventDef.getTimestamp();
+        final CtfTmfTimestamp timestamp = new CtfTmfTimestamp(
+                originTrace.getCTFTrace().timestampCyclesToNanos(ts));
 
         int sourceCPU = eventDef.getCPU();
 
@@ -65,16 +70,43 @@ public final class CtfTmfEventFactory {
 
         String reference = fileName == null ? CtfTmfEvent.NO_STREAM : fileName;
 
-        /* Construct and return the object */
+        /* Handle the special case of lost events */
+        if (eventDecl.getName().equals(CTFStrings.LOST_EVENT_NAME)) {
+            Definition nbLostEventsDef = eventDef.getFields().getDefinitions().get(CTFStrings.LOST_EVENTS_FIELD);
+            Definition durationDef = eventDef.getFields().getDefinitions().get(CTFStrings.LOST_EVENTS_DURATION);
+            if (!(nbLostEventsDef instanceof IntegerDefinition) || !(durationDef instanceof IntegerDefinition)) {
+                /*
+                 * One or both of these fields doesn't exist, or is not of the
+                 * right type. The event claims to be a "lost event", but is
+                 * malformed. Log it and return a null event instead.
+                 */
+                return getNullEvent();
+            }
+            long nbLostEvents = ((IntegerDefinition) nbLostEventsDef).getValue();
+            long duration = ((IntegerDefinition) durationDef).getValue();
+            CtfTmfTimestamp timestampEnd = new CtfTmfTimestamp(
+                    originTrace.getCTFTrace().timestampCyclesToNanos(ts) + duration);
+
+            CtfTmfLostEvent lostEvent = new CtfTmfLostEvent(originTrace,
+                    ITmfContext.UNKNOWN_RANK,
+                    content,
+                    reference, // filename
+                    sourceCPU,
+                    eventDecl,
+                    new TmfTimeRange(timestamp, timestampEnd),
+                    nbLostEvents);
+            return lostEvent;
+        }
+
+        /* Handle standard event types */
         CtfTmfEvent event = new CtfTmfEvent(
                 originTrace,
                 ITmfContext.UNKNOWN_RANK,
                 timestamp,
                 content,
-                reference,
+                reference, // filename
                 sourceCPU,
-                eventDef.getDeclaration()
-        );
+                eventDecl);
         return event;
     }
 
diff --git a/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLostEvent.java b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLostEvent.java
new file mode 100644
index 0000000000..3c7ac043c1
--- /dev/null
+++ b/lttng/org.eclipse.linuxtools.tmf.core/src/org/eclipse/linuxtools/tmf/core/ctfadaptor/CtfTmfLostEvent.java
@@ -0,0 +1,79 @@
+/*******************************************************************************
+ * Copyright (c) 2013 Ericsson
+ *
+ * All rights reserved. This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License v1.0 which
+ * accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *   Alexandre Montplaisir - Initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.linuxtools.tmf.core.ctfadaptor;
+
+import org.eclipse.linuxtools.ctf.core.event.IEventDeclaration;
+import org.eclipse.linuxtools.tmf.core.event.ITmfEventField;
+import org.eclipse.linuxtools.tmf.core.event.ITmfLostEvent;
+import org.eclipse.linuxtools.tmf.core.timestamp.TmfTimeRange;
+
+/**
+ * An implementation of {@link ITmfLostEvent} for use in the CTF adaptor.
+ *
+ * @author Alexandre Montplaisir
+ * @since 2.1
+ */
+public class CtfTmfLostEvent extends CtfTmfEvent implements ITmfLostEvent {
+
+    private final TmfTimeRange fTimeRange;
+    private final long fNbLost;
+
+    /**
+     * Constructor. Only {@link CtfTmfEventFactory} should call this.
+     *
+     * @param trace
+     *            The origin trace
+     * @param rank
+     *            The rank of the event in the trace
+     * @param content
+     *            The event's payload (fields). In case this event has some.
+     * @param fileName
+     *            The name of the trace file from which this event comes
+     * @param cpu
+     *            The CPU on which this event happend
+     * @param declaration
+     *            The CTF Event Declaration object that created this event
+     * @param timeRange
+     *            The time range of lost events indicated by this one
+     * @param nbLost
+     *            The number of lost events in the range
+     */
+    CtfTmfLostEvent(CtfTmfTrace trace,
+            long rank,
+            ITmfEventField content,
+            String fileName,
+            int cpu,
+            IEventDeclaration declaration,
+            TmfTimeRange timeRange,
+            long nbLost) {
+        /*
+         * Only the factory should call this method, the case to
+         * (CtfTmfTimestamp) should be safe.
+         */
+        super(trace, rank, (CtfTmfTimestamp) timeRange.getStartTime(), content, fileName, cpu, declaration);
+        fTimeRange = timeRange;
+        fNbLost = nbLost;
+    }
+
+    @Override
+    public TmfTimeRange getTimeRange() {
+        return fTimeRange;
+    }
+
+    @Override
+    public long getNbLostEvents() {
+        return fNbLost;
+    }
+
+
+}