Bug 534923 - Cocoa: objc_msgSend is called incorrectly from os.c

C calling conventions may be different for vararg and non-vararg
functions. Change the generated code to not cast vararg functions
to function pointers with non-vararg profiles. Instead annotate
the objc_msgSend* functions in MacGenerator as "fixedargs=2" (meaning:
it's a vararg function with two fixed parameters), and change the
NativesGenerator to use this annotation to generate a cast to a
function pointer with a vararg profile with that many fixed
parameters. "fixedargs=0" or absence of the "fixedargs" annotation
means all parameters are fixed; there are no varargs. The behavior
in this case in unchanged.

The casting cannot be avoided altogether; the objc_msgSend*_stret
functions _must_ be casted.

Also use BOOL as return type for the cast of objc_msgSendSuper_bool,
not just for objc_msgSend_bool.

Verified that casting to a profile with varargs indeed makes the
compiler generate the assembly code that on x86_64 does clear
register %al, as it should per the ABI spec.[1] The old code
generated did not do this. (Verified with with cc -S using Apple's
clang and inspecting the assembler output; on OS X 10.14.2.)

Ran the new MacGenerator to regenerate the Cocoa OS.java, then ran
the new JNIGenerator to regenerate the Cocoa os.c. Re-built the
binary library. Set up an Eclipse using this rebuilt library and
ran all SWT tests. All tests passed locally.

[1] https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf

Change-Id: Ib03226ad9ecdf4bfcf789cd2b42f592b9a2b9021
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

2 files changed, 36 insertions, 6 deletions

diff --git a/bundles/org.eclipse.swt.tools/JNI Generation/org/eclipse/swt/tools/internal/NativesGenerator.java b/bundles/org.eclipse.swt.tools/JNI Generation/org/eclipse/swt/tools/internal/NativesGenerator.java
index 5816ce8fdb..c05290e1ee 100644
--- a/bundles/org.eclipse.swt.tools/JNI Generation/org/eclipse/swt/tools/internal/NativesGenerator.java
+++ b/bundles/org.eclipse.swt.tools/JNI Generation/org/eclipse/swt/tools/internal/NativesGenerator.java
@@ -11,7 +11,8 @@
  * Contributors:
  *     IBM Corporation - initial API and implementation
  *     Red Hat Inc. - stop generating pre 1.2 JNI code
- * Martin Oberhuber (WindRiver) - [515610] Fix incorrect code for memmove()
+ *     Martin Oberhuber (WindRiver) - [515610] Fix incorrect code for memmove()
+ *     Thomas Wolf <thomas.wolf@paranor.ch> [534923] Fix incorrect casts
  *******************************************************************************/
 package org.eclipse.swt.tools.internal;
 
@@ -793,12 +794,14 @@ void generateFunctionCall(JNIMethod method, JNIParameter[] params, JNIType retur
 		if (method.getFlag(Flags.FLAG_CAST)) {
 			output("((");
 			String returnCast = returnType.getTypeSignature2(!returnType.equals(returnType64));
-			if (name.equals("objc_msgSend_bool") && returnCast.equals("jboolean")) {
+			if ((name.equals("objc_msgSend_bool") || name.equals("objc_msgSendSuper_bool")) && returnCast.equals("jboolean")) {
 				returnCast = "BOOL";
 			}
 			output(returnCast);
 			output(" (*)(");
-			for (int i = 0; i < params.length; i++) {
+			int fixedargs = getNumberOfFixedArguments(method);
+			int n = fixedargs > 0 ? fixedargs : params.length;
+			for (int i = 0; i < n; i++) {
 				if (i != 0) output(", ");
 				JNIParameter param = params[i];
 				String cast = param.getCast();
@@ -816,6 +819,9 @@ void generateFunctionCall(JNIMethod method, JNIParameter[] params, JNIType retur
 					output(paramType.getTypeSignature4(!paramType.equals(paramType64), param.getFlag(FLAG_STRUCT)));
 				}
 			}
+			if (fixedargs > 0) {
+				output(", ...");
+			}
 			output("))");
 		}
 		String accessor = method.getAccessor();
@@ -875,7 +881,8 @@ void generate_objc_msgSend_stret (JNIParameter[] params, String func) {
 	JNIType paramType = params[0].getType(), paramType64 = params[0].getType64();
 	output(paramType.getTypeSignature4(!paramType.equals(paramType64), true));
 	output(" (*)(");
-	for (int i = 1; i < params.length; i++) {
+	// Only the fixed arguments
+	for (int i = 1; i <= 2; i++) {
 		if (i != 1) output(", ");
 		JNIParameter param = params[i];
 		String cast = param.getCast();
@@ -893,11 +900,33 @@ void generate_objc_msgSend_stret (JNIParameter[] params, String func) {
 			output(paramType.getTypeSignature4(!paramType.equals(paramType64), param.getFlag(FLAG_STRUCT)));
 		}
 	}
+	output(", ...");
 	output("))");
 	output(func);
 	output(")");
 }
 
+/**
+ * Determines whether the {@code method} has variadic arguments. If so, returns
+ * the number of fixed parameters (always {@code > 0}). Otherwise returns zero,
+ * which indicates that all parameters are fixed.
+ * 
+ * @param method to examine
+ * @return the number of fixed parameters, or zero if all parameters are fixed
+ *         (the method has no varargs)
+ */
+int getNumberOfFixedArguments(JNIMethod method) {
+	Object param = method.getParam("fixedargs");
+	if (param == null) {
+		return 0;
+	}
+	String number = param.toString();
+	if (number.isEmpty()) {
+		return 0;
+	}
+	return Integer.parseInt(number);
+}
+
 void generateReturn(JNIType returnType, boolean needsReturn) {
 	if (needsReturn && !returnType.isType("void")) {
 		outputln("\treturn rc;");
diff --git a/bundles/org.eclipse.swt.tools/Mac Generation/org/eclipse/swt/tools/internal/MacGenerator.java b/bundles/org.eclipse.swt.tools/Mac Generation/org/eclipse/swt/tools/internal/MacGenerator.java
index 90cf587667..4c73931ff6 100644
--- a/bundles/org.eclipse.swt.tools/Mac Generation/org/eclipse/swt/tools/internal/MacGenerator.java
+++ b/bundles/org.eclipse.swt.tools/Mac Generation/org/eclipse/swt/tools/internal/MacGenerator.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2008, 2017 IBM Corporation and others.
+ * Copyright (c) 2008, 2019 IBM Corporation and others.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -10,6 +10,7 @@
  *
  * Contributors:
  *     IBM Corporation - initial API and implementation
+ *     Thomas Wolf <thomas.wolf@paranor.ch> [534923] Annotate sends "fixedargs=2"
  *******************************************************************************/
 package org.eclipse.swt.tools.internal;
 
@@ -1742,7 +1743,7 @@ void generateSends(boolean superCall) {
 			outln();
 			out(" *");
 		}
-		out(" @method flags=cast");
+		out(" @method flags=cast,fixedargs=2");
 		if (tags.size() > 0) outln();
 		for (String tag : tags) {
 			out(tag);