traceEvent: redirect metadata events to the properties.

This removes events with a typically 0 timestamp from the traace.
It makes the trace visualization much more useful, and keeps
all the (timeless) metadata information in the properties.

Also make trace into a thread name and process name provider and
update analyses to use it.

Change-Id: Ibad29991bba33e97854917ced84016d070e5ad8d
Signed-off-by: Matthew Khouzam <matthew.khouzam@ericsson.com>
Reviewed-on: https://git.eclipse.org/r/119146
Tested-by: CI Bot
Reviewed-by: Genevieve Bastien <gbastien+lttng@versatic.net>

7 files changed, 224 insertions, 24 deletions

diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core.tests/src/org/eclipse/tracecompass/incubator/traceevent/core/tests/TraceEventTraceTest.java b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core.tests/src/org/eclipse/tracecompass/incubator/traceevent/core/tests/TraceEventTraceTest.java
index 1710896da..108a8b642 100644
--- a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core.tests/src/org/eclipse/tracecompass/incubator/traceevent/core/tests/TraceEventTraceTest.java
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core.tests/src/org/eclipse/tracecompass/incubator/traceevent/core/tests/TraceEventTraceTest.java
@@ -15,6 +15,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 import org.eclipse.jdt.annotation.NonNull;
@@ -22,6 +23,7 @@ import org.eclipse.tracecompass.incubator.internal.traceevent.core.trace.TraceEv
 import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
 import org.eclipse.tracecompass.tmf.core.event.aspect.ITmfEventAspect;
 import org.eclipse.tracecompass.tmf.core.exceptions.TmfTraceException;
+import org.eclipse.tracecompass.tmf.core.project.model.ITmfPropertiesProvider;
 import org.eclipse.tracecompass.tmf.core.timestamp.ITmfTimestamp;
 import org.eclipse.tracecompass.tmf.core.timestamp.TmfTimestamp;
 import org.eclipse.tracecompass.tmf.core.trace.ITmfContext;
@@ -50,10 +52,10 @@ public class TraceEventTraceTest {
         Map<String, ITmfEventAspect<?>> eventAspects = getEventAspects(path);
         ITmfEvent event = getFirstEvent(path);
         assertNotNull(event);
-        ImmutableSet<String> aspectNames = ImmutableSet.of("Thread ID", "Args", "Phase", "Category", "PID", "Duration", "ID", "Callsite", "Timestamp", "LogLevel", "Name");
+        ImmutableSet<String> aspectNames = ImmutableSet.of("TID", "Args", "Phase", "Category", "PID", "Duration", "ID", "Callsite", "Timestamp", "LogLevel", "Name", "Process Name", "Thread Name");
 
         assertEquals(aspectNames, eventAspects.keySet());
-        testAspect(eventAspects.get("Thread ID"), event, 0);
+        testAspect(eventAspects.get("TID"), event, 0);
         testAspect(eventAspects.get("Phase"), event, "C");
         testAspect(eventAspects.get("Category"), event, null);
         testAspect(eventAspects.get("Name"), event, "foo");
@@ -139,14 +141,44 @@ public class TraceEventTraceTest {
      */
     @Test
     public void testChromeosTrace() throws TmfTraceException {
+        String[] env = { "Type", "Trace-Event",
+                "process_sort_index-5044", "-1",
+                "pid-5044", "GPU Process",
+                "tid-5051", "Chrome_ChildIOThread",
+                "process_sort_index-5075", "-5",
+                "pid-5075", "Renderer",
+                "pidLabel-5075", "chrome://tracing",
+                "thread_sort_index-12", "-1",
+                "tid-13", "Chrome_ChildIOThread",
+                "process_sort_index-5243", "-5",
+                "pid-5243", "Renderer",
+                "pidLabel-5243", "The New York Times - Breaking News, World News & Multimedia",
+                "thread_sort_index-73", "-1",
+                "process_sort_index-5145", "-5",
+                "pid-5145", "Renderer",
+                "pidLabel-5145", "The New York Times - Breaking News, World News & Multimedia",
+                "thread_sort_index-27", "-1",
+                "process_sort_index-5173", "-5",
+                "pid-5173", "Renderer",
+                "pidLabel-5173", "The New York Times - Breaking News, World News & Multimedia",
+                "thread_sort_index-43", "-1",
+                "process_sort_index-5014", "-6",
+                "pid-5014", "Browser",
+                "tid-5036", "Chrome_IOThread",
+                "tid-5014", "CrBrowserMain" };
+        Map<String, String> expectedProperties = new LinkedHashMap<>();
+        for (int i = 0; i < env.length; i += 2) {
+            expectedProperties.put(env[i], env[i + 1]);
+        }
         String path = "traces/chromeos_system_trace.json";
-        int nbEvents = 36;
-        ITmfTimestamp startTime = TmfTimestamp.fromNanos(0);
+        int nbEvents = 12;
+        ITmfTimestamp startTime = TmfTimestamp.fromMicros(5443650636079L);
         ITmfTimestamp endTime = TmfTimestamp.fromMicros(5443672642443L);
-        testTrace(path, nbEvents, startTime, endTime);
+        Map<String, String> properties = testTrace(path, nbEvents, startTime, endTime);
+        assertEquals(expectedProperties, properties);
     }
 
-    private static void testTrace(String path, int nbEvents, ITmfTimestamp startTime, ITmfTimestamp endTime) throws TmfTraceException {
+    private static Map<String, String> testTrace(String path, int nbEvents, ITmfTimestamp startTime, ITmfTimestamp endTime) throws TmfTraceException {
         ITmfTrace trace = new TraceEventTrace();
         try {
             assertTrue(trace.validate(null, path).isOK());
@@ -168,6 +200,8 @@ public class TraceEventTraceTest {
             assertEquals(nbEvents, trace.getNbEvents());
             assertEquals(startTime.toNanos(), trace.getStartTime().toNanos());
             assertEquals(endTime.toNanos(), trace.getEndTime().toNanos());
+            assertTrue(trace instanceof ITmfPropertiesProvider);
+            return ((ITmfPropertiesProvider) trace).getProperties();
         } finally {
             trace.dispose();
         }
diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter.png b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter.png
new file mode 100644
index 000000000..532b1cf9d
--- /dev/null
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter.png
diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter@2x.png b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter@2x.png
new file mode 100644
index 000000000..cd42e2085
--- /dev/null
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter@2x.png
diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter@4x.png b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter@4x.png
new file mode 100644
index 000000000..e216c114a
--- /dev/null
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/icons/counter@4x.png
diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/plugin.xml b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/plugin.xml
index 0d9de88c5..ecb0e6c80 100644
--- a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/plugin.xml
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/plugin.xml
@@ -32,6 +32,7 @@
       <module
             analysis_module="org.eclipse.tracecompass.incubator.internal.traceevent.core.analysis.counter.TraceEventCounterAnalysis"
             automatic="true"
+            icon="icons/counter.png"
             id="org.eclipse.tracecompass.incubator.traceevent.core.counter"
             name="%counter.name">
          <tracetype
diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/event/TraceEventAspects.java b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/event/TraceEventAspects.java
index 3bcd33049..980784a82 100644
--- a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/event/TraceEventAspects.java
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/event/TraceEventAspects.java
@@ -14,6 +14,8 @@ import java.util.logging.Level;
 
 import org.eclipse.jdt.annotation.NonNull;
 import org.eclipse.jdt.annotation.Nullable;
+import org.eclipse.tracecompass.analysis.os.linux.core.event.aspect.LinuxTidAspect;
+import org.eclipse.tracecompass.tmf.core.event.ITmfEvent;
 import org.eclipse.tracecompass.tmf.core.event.aspect.ITmfEventAspect;
 import org.eclipse.tracecompass.tmf.core.event.aspect.TmfBaseAspects;
 import org.eclipse.tracecompass.tmf.core.event.lookup.ITmfCallsite;
@@ -135,25 +137,28 @@ public class TraceEventAspects {
         }
     }
 
-    private static class TraceCompassScopeLogTidAspect implements ITraceEventAspect<Integer> {
+    private static class TraceCompassScopeLogTidAspect extends LinuxTidAspect {
 
         @Override
-        public @NonNull String getName() {
-            return String.valueOf(Messages.TraceCompassScopeLogAspects_ThreadId);
-        }
-
-        @Override
-        public @NonNull String getHelpText() {
-            return String.valueOf(Messages.TraceCompassScopeLogAspects_ThreadIdD);
+        public boolean isHiddenByDefault() {
+            return false;
         }
 
         @Override
-        public @Nullable Integer resolveTCL(@NonNull TraceEventEvent event) {
-            return event.getField().getTid();
+        public @Nullable Integer resolve(@NonNull ITmfEvent event) {
+            if (event instanceof TraceEventEvent) {
+                return ((TraceEventEvent) event).getField().getTid();
+            }
+            return null;
         }
     }
 
-    private static class TraceCompassScopeLogPidAspect implements ITraceEventAspect<String> {
+    /**
+     * Numerical PID aspect
+     *
+     * @author Matthew Khouzam
+     */
+    public static class TraceCompassScopeLogPidAspect implements ITraceEventAspect<String> {
 
         @Override
         public @NonNull String getName() {
diff --git a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/trace/TraceEventTrace.java b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/trace/TraceEventTrace.java
index 0d9769b03..716cb791f 100644
--- a/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/trace/TraceEventTrace.java
+++ b/tracetypes/org.eclipse.tracecompass.incubator.traceevent.core/src/org/eclipse/tracecompass/incubator/internal/traceevent/core/trace/TraceEventTrace.java
@@ -14,8 +14,12 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
 
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.IResource;
@@ -44,6 +48,8 @@ import org.eclipse.tracecompass.tmf.core.trace.indexer.ITmfPersistentlyIndexable
 import org.eclipse.tracecompass.tmf.core.trace.location.ITmfLocation;
 import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
 
+import com.google.common.collect.Lists;
+
 /**
  * Trace event trace. Can read trace event unsorted or sorted JSON traces.
  *
@@ -52,16 +58,75 @@ import org.eclipse.tracecompass.tmf.core.trace.location.TmfLongLocation;
  */
 public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexable, ITmfPropertiesProvider, ITmfTraceKnownSize {
 
+    /**
+     * Tid prefix to add to thread name
+     */
+    private static final String TID_PREFIX = "tid-"; //$NON-NLS-1$
+    /**
+     * Pid Labels prefix
+     */
+    private static final String PID_LABEL_PREFIX = "pidLabel-"; //$NON-NLS-1$
+    /**
+     * Pid name prefix
+     */
+    private static final String PID_PREFIX = "pid-"; //$NON-NLS-1$
+    /**
+     * Metadata Field String Name
+     */
+    private static final String NAME_ARG = "name"; //$NON-NLS-1$
+    /**
+     * Metadata Field String labels
+     */
+    private static final String LABELS = "labels"; //$NON-NLS-1$
+    /**
+     * Metadata Field String sort index
+     */
+    private static final String SORT_INDEX = "sort_index"; //$NON-NLS-1$
+    /**
+     * Metadata String Process Name
+     */
+    private static final String PROCESS_NAME = "process_name"; //$NON-NLS-1$
+    /**
+     * Metadata String Process Labels
+     */
+    private static final String PROCESS_LABELS = "process_labels"; //$NON-NLS-1$
+    /**
+     * Metadata String Process Sort Index
+     */
+    private static final String PROCESS_SORT_INDEX = "process_sort_index"; //$NON-NLS-1$
+    /**
+     * Metadata String Thread Name
+     */
+    private static final String THREAD_NAME = "thread_name"; //$NON-NLS-1$
+    /**
+     * Metadata String Thread Sort Index
+     */
+    private static final String THREAD_SORT_INDEX = "thread_sort_index"; //$NON-NLS-1$
+
     private static final int CHECKPOINT_SIZE = 10000;
     private static final int ESTIMATED_EVENT_SIZE = 90;
     private static final TmfLongLocation NULL_LOCATION = new TmfLongLocation(-1L);
     private static final TmfContext INVALID_CONTEXT = new TmfContext(NULL_LOCATION, ITmfContext.UNKNOWN_RANK);
     private static final int MAX_LINES = 100;
     private static final int MAX_CONFIDENCE = 100;
+    private final @NonNull Map<@NonNull String, @NonNull String> fProperties = new LinkedHashMap<>();
+    private final @NonNull Map<Object, String> fPidNames = new HashMap<>();
+    private final @NonNull NavigableMap<Integer, String> fTidNames = new TreeMap<>();
 
     private File fFile;
 
     private RandomAccessFile fFileInput;
+    private final @NonNull Iterable<@NonNull ITmfEventAspect<?>> fEventAspects;
+
+    /**
+     * Constructor
+     */
+    public TraceEventTrace() {
+        List<@NonNull ITmfEventAspect<?>> aspects = Lists.newArrayList(TraceEventAspects.getAspects());
+        aspects.add(new ProcessNameAspect());
+        aspects.add(new ThreadNameAspect());
+        fEventAspects = aspects;
+    }
 
     @Override
     public IStatus validate(IProject project, String path) {
@@ -73,6 +138,7 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Not a file. It's a directory: " + path); //$NON-NLS-1$
         }
         int confidence = 0;
+
         try {
             if (!TmfTraceUtils.isText(file)) {
                 return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
@@ -82,6 +148,7 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
             return new Status(IStatus.ERROR, Activator.PLUGIN_ID, "IOException validating file: " + path, e); //$NON-NLS-1$
         }
         try (BufferedRandomAccessFile rafile = new BufferedRandomAccessFile(path, "r")) { //$NON-NLS-1$
+            goToCorrectStart(rafile);
             int lineCount = 0;
             int matches = 0;
             String line = readNextEventString(() -> (char) rafile.read());
@@ -108,9 +175,19 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
         return new TraceValidationStatus(confidence, Activator.PLUGIN_ID);
     }
 
+    private static void goToCorrectStart(RandomAccessFile rafile) throws IOException {
+        // skip start if it's {"traceEvents":
+        if (rafile.readLine().startsWith("{\"traceEvents\":")) { //$NON-NLS-1$
+            rafile.seek(14);
+        } else {
+            rafile.seek(0);
+        }
+    }
+
     @Override
     public void initTrace(IResource resource, String path, Class<? extends ITmfEvent> type) throws TmfTraceException {
         super.initTrace(resource, path, type);
+        fProperties.put("Type", "Trace-Event"); //$NON-NLS-1$ //$NON-NLS-2$ , value)
         String dir = TmfTraceManager.getSupplementaryFileDir(this);
         fFile = new File(dir + new File(path).getName());
         if (!fFile.exists()) {
@@ -128,6 +205,7 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
         }
         try {
             fFileInput = new BufferedRandomAccessFile(fFile, "r"); //$NON-NLS-1$
+            goToCorrectStart(fFileInput);
         } catch (IOException e) {
             throw new TmfTraceException(e.getMessage(), e);
         }
@@ -191,7 +269,7 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
 
     @Override
     public Iterable<@NonNull ITmfEventAspect<?>> getEventAspects() {
-        return TraceEventAspects.getAspects();
+        return fEventAspects;
     }
 
     @Override
@@ -210,9 +288,16 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
                         fFileInput.seek(locationInfo);
                     }
                     String nextJson = readNextEventString(() -> (char) fFileInput.read());
-                    if (nextJson != null) {
+                    while (nextJson != null) {
                         TraceEventField field = TraceEventField.parseJson(nextJson);
-                        return new TraceEventEvent(this, context.getRank(), field);
+                        if (field == null) {
+                            return null;
+                        }
+                        if (field.getPhase() != 'M') {
+                            return new TraceEventEvent(this, context.getRank(), field);
+                        }
+                        parseMetadata(field);
+                        nextJson = readNextEventString(() -> (char) fFileInput.read());
                     }
                 } catch (IOException e) {
                     Activator.getInstance().logError("Error parsing event", e); //$NON-NLS-1$
@@ -222,6 +307,52 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
         return null;
     }
 
+    private void parseMetadata(TraceEventField field) {
+        Map<@NonNull String, @NonNull Object> args = field.getArgs();
+        String name = field.getName();
+        if (args == null) {
+            return;
+        }
+        switch (name) {
+        case PROCESS_NAME:
+            String procName = (String) args.get(NAME_ARG);
+            fPidNames.put(field.getPid(), procName);
+            if (procName != null) {
+                fProperties.put(PID_PREFIX + field.getPid(), procName);
+            }
+            break;
+        case PROCESS_LABELS:
+            String procLabels = (String) args.get(LABELS);
+            if (procLabels != null) {
+                fProperties.put(PID_LABEL_PREFIX + field.getPid(), procLabels);
+            }
+            break;
+        case PROCESS_SORT_INDEX:
+            String sortIndex = (String) args.get(SORT_INDEX);
+            if (sortIndex != null) {
+                fProperties.put(name + '-' + field.getPid(), sortIndex);
+            }
+            break;
+        case THREAD_NAME:
+            String threadName = (String) args.get(NAME_ARG);
+            fTidNames.put(field.getTid(), threadName);
+            if (threadName != null) {
+                fProperties.put(TID_PREFIX + field.getTid(), threadName);
+            }
+            break;
+        case THREAD_SORT_INDEX:
+            sortIndex = (String) args.get(SORT_INDEX);
+            if (sortIndex != null) {
+                fProperties.put(name + '-' + field.getTid(), sortIndex);
+            }
+            break;
+        default:
+            fProperties.put(name, String.valueOf(args));
+            break;
+        }
+
+    }
+
     @Override
     public ITmfLocation getCurrentLocation() {
         long temp = -1;
@@ -234,7 +365,7 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
 
     @Override
     public @NonNull Map<@NonNull String, @NonNull String> getProperties() {
-        return Collections.singletonMap("Type", "Trace-Event"); //$NON-NLS-1$ //$NON-NLS-2$
+        return fProperties;
     }
 
     @Override
@@ -248,8 +379,8 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
     }
 
     /**
-     * Wrapper to get a character reader, allows to reconcile between java.nio
-     * and java.io
+     * Wrapper to get a character reader, allows to reconcile between java.nio and
+     * java.io
      *
      * @author Matthew Khouzam
      *
@@ -343,4 +474,33 @@ public class TraceEventTrace extends TmfTrace implements ITmfPersistentlyIndexab
         return length > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) length;
     }
 
+    /**
+     * Get the Process name
+     */
+    public class ProcessNameAspect extends org.eclipse.tracecompass.incubator.analysis.core.aspects.ProcessNameAspect {
+
+        @Override
+        public @Nullable String resolve(@NonNull ITmfEvent event) {
+            if (event instanceof TraceEventEvent) {
+                TraceEventField field = ((TraceEventEvent) event).getField();
+                return fPidNames.get(field.getPid());
+            }
+            return null;
+        }
+    }
+
+    /**
+     * Get the Thread name
+     */
+    public class ThreadNameAspect extends org.eclipse.tracecompass.incubator.analysis.core.aspects.ThreadNameAspect {
+
+        @Override
+        public @Nullable String resolve(@NonNull ITmfEvent event) {
+            if (event instanceof TraceEventEvent) {
+                TraceEventField field = ((TraceEventEvent) event).getField();
+                return fTidNames.get(field.getTid());
+            }
+            return null;
+        }
+    }
 }