Bug 568251 - Prevent the error from happening again

Please refer to new code comment in 'swt.h' for explanation.

Fix use of 'jbooleanArray' in 'WideCharToMultiByte()' on Windows.
Luckily, all uses currently pass null to the parameter, so the problem
doesn't trigger in current SWT.

I already saw the problem of 'jbooleanArray' three times recently:
1) Bug 568251 in GTK
2) This problem in 'WideCharToMultiByte()'
3) When reviewing Bug 563282.

Also, the problem is far from being obvious. Therefore, it sounds like
a good idea to add a code guard to prevent the problem from
ever happening again. A new #define in 'swt.h' will serve this purpose.

Change-Id: I4c9af55a6de41332f797d1d6200ebe1f83abd353
Signed-off-by: Alexandr Miloslavskiy <alexandr.miloslavskiy@syntevo.com>

6 files changed, 61 insertions, 39 deletions

diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/gtk/org/eclipse/swt/internal/gtk/OS.java b/bundles/org.eclipse.swt/Eclipse SWT PI/gtk/org/eclipse/swt/internal/gtk/OS.java
index 68a1e9dd29..142b40ef89 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/gtk/org/eclipse/swt/internal/gtk/OS.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/gtk/org/eclipse/swt/internal/gtk/OS.java
@@ -20,32 +20,38 @@ import java.util.*;
 import org.eclipse.swt.internal.*;
 
 // Common type translation table:
-// C   ->  Java
+// C            ->  Java
 // --------------------
 // Primitives:
-// int   -> int
-// guint -> long   #Reason:
-//					c : unsigned int range: 4294967295
-//                  java : int range      : 2147483647 (less than c unsigned int)
-//                  Java : long range: 9,223,372,036,854,775,807
-//				    // Note: Not to be used for pointers.
+// int          -> int
+// gint*        -> int[]
 //
-// gint* -> int[]
-// boolean   -> int  ex setenv
-// gboolean  -> boolean
+// Unsigned integer:
+// * Note that java's int is signed, which introduces difficulties
+// * for values > 0x7FFFFFFF. Java's long can fit such values, but
+// * java's long is 8 bytes, while guint is 4 bytes. For that reason,
+// * java's long CAN'T be used for pointers or arrays.
+// guint        -> int/long
+// guint*       -> int[]
+//
+// Boolean:
+// * Java's boolean is handy, but it's 1 byte, while gboolean is 4
+// * bytes. For that reason, it CAN'T be used for pointers or arrays.
+// gboolean     -> int/boolean
+// gboolean*    -> int
 //
 // Pointers:
-// gpointer -> long
-// void *   -> long
+// gpointer     -> long
+// void *       -> long
 //
 // Strings:
-// gchar *      -> long
-// const char * -> byte[]  ex setenv
-// const gchar* -> byte[]  ex g_log_remove_handler
+// gchar *      -> long    // You're responsible for allocating/deallocating memory buffer.
+// const char * -> byte[]  // Example: setenv()
+// const gchar* -> byte[]  // Example: g_log_remove_handler()
 //
 // Special types:
-// GQuark -> int
-// GError ** -> long []  ex g_filename_to_uri
+// GQuark       -> int
+// GError **    -> long[]  // Example: g_filename_to_uri()
 
 
 /**
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os.c b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os.c
index c46c8de646..9a95393e6d 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os.c
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os.c
@@ -9964,49 +9964,49 @@ JNIEXPORT jboolean JNICALL OS_NATIVE(WaitMessage)
 }
 #endif
 
-#ifndef NO_WideCharToMultiByte__II_3CIJI_3B_3Z
-JNIEXPORT jint JNICALL OS_NATIVE(WideCharToMultiByte__II_3CIJI_3B_3Z)
-	(JNIEnv *env, jclass that, jint arg0, jint arg1, jcharArray arg2, jint arg3, jlong arg4, jint arg5, jbyteArray arg6, jbooleanArray arg7)
+#ifndef NO_WideCharToMultiByte__II_3CIJI_3B_3I
+JNIEXPORT jint JNICALL OS_NATIVE(WideCharToMultiByte__II_3CIJI_3B_3I)
+	(JNIEnv *env, jclass that, jint arg0, jint arg1, jcharArray arg2, jint arg3, jlong arg4, jint arg5, jbyteArray arg6, jintArray arg7)
 {
 	jchar *lparg2=NULL;
 	jbyte *lparg6=NULL;
-	jboolean *lparg7=NULL;
+	jint *lparg7=NULL;
 	jint rc = 0;
-	OS_NATIVE_ENTER(env, that, WideCharToMultiByte__II_3CIJI_3B_3Z_FUNC);
+	OS_NATIVE_ENTER(env, that, WideCharToMultiByte__II_3CIJI_3B_3I_FUNC);
 	if (arg6) if ((lparg6 = (*env)->GetByteArrayElements(env, arg6, NULL)) == NULL) goto fail;
-	if (arg7) if ((lparg7 = (*env)->GetBooleanArrayElements(env, arg7, NULL)) == NULL) goto fail;
+	if (arg7) if ((lparg7 = (*env)->GetIntArrayElements(env, arg7, NULL)) == NULL) goto fail;
 		if (arg2) if ((lparg2 = (*env)->GetPrimitiveArrayCritical(env, arg2, NULL)) == NULL) goto fail;
 	rc = (jint)WideCharToMultiByte(arg0, arg1, (LPCWSTR)lparg2, arg3, (LPSTR)arg4, arg5, (LPCSTR)lparg6, (LPBOOL)lparg7);
 fail:
 		if (arg2 && lparg2) (*env)->ReleasePrimitiveArrayCritical(env, arg2, lparg2, JNI_ABORT);
-	if (arg7 && lparg7) (*env)->ReleaseBooleanArrayElements(env, arg7, lparg7, 0);
+	if (arg7 && lparg7) (*env)->ReleaseIntArrayElements(env, arg7, lparg7, 0);
 	if (arg6 && lparg6) (*env)->ReleaseByteArrayElements(env, arg6, lparg6, 0);
-	OS_NATIVE_EXIT(env, that, WideCharToMultiByte__II_3CIJI_3B_3Z_FUNC);
+	OS_NATIVE_EXIT(env, that, WideCharToMultiByte__II_3CIJI_3B_3I_FUNC);
 	return rc;
 }
 #endif
 
-#ifndef NO_WideCharToMultiByte__II_3CI_3BI_3B_3Z
-JNIEXPORT jint JNICALL OS_NATIVE(WideCharToMultiByte__II_3CI_3BI_3B_3Z)
-	(JNIEnv *env, jclass that, jint arg0, jint arg1, jcharArray arg2, jint arg3, jbyteArray arg4, jint arg5, jbyteArray arg6, jbooleanArray arg7)
+#ifndef NO_WideCharToMultiByte__II_3CI_3BI_3B_3I
+JNIEXPORT jint JNICALL OS_NATIVE(WideCharToMultiByte__II_3CI_3BI_3B_3I)
+	(JNIEnv *env, jclass that, jint arg0, jint arg1, jcharArray arg2, jint arg3, jbyteArray arg4, jint arg5, jbyteArray arg6, jintArray arg7)
 {
 	jchar *lparg2=NULL;
 	jbyte *lparg4=NULL;
 	jbyte *lparg6=NULL;
-	jboolean *lparg7=NULL;
+	jint *lparg7=NULL;
 	jint rc = 0;
-	OS_NATIVE_ENTER(env, that, WideCharToMultiByte__II_3CI_3BI_3B_3Z_FUNC);
+	OS_NATIVE_ENTER(env, that, WideCharToMultiByte__II_3CI_3BI_3B_3I_FUNC);
 	if (arg6) if ((lparg6 = (*env)->GetByteArrayElements(env, arg6, NULL)) == NULL) goto fail;
-	if (arg7) if ((lparg7 = (*env)->GetBooleanArrayElements(env, arg7, NULL)) == NULL) goto fail;
+	if (arg7) if ((lparg7 = (*env)->GetIntArrayElements(env, arg7, NULL)) == NULL) goto fail;
 		if (arg2) if ((lparg2 = (*env)->GetPrimitiveArrayCritical(env, arg2, NULL)) == NULL) goto fail;
 		if (arg4) if ((lparg4 = (*env)->GetPrimitiveArrayCritical(env, arg4, NULL)) == NULL) goto fail;
 	rc = (jint)WideCharToMultiByte(arg0, arg1, (LPCWSTR)lparg2, arg3, (LPSTR)lparg4, arg5, (LPCSTR)lparg6, (LPBOOL)lparg7);
 fail:
 		if (arg4 && lparg4) (*env)->ReleasePrimitiveArrayCritical(env, arg4, lparg4, 0);
 		if (arg2 && lparg2) (*env)->ReleasePrimitiveArrayCritical(env, arg2, lparg2, JNI_ABORT);
-	if (arg7 && lparg7) (*env)->ReleaseBooleanArrayElements(env, arg7, lparg7, 0);
+	if (arg7 && lparg7) (*env)->ReleaseIntArrayElements(env, arg7, lparg7, 0);
 	if (arg6 && lparg6) (*env)->ReleaseByteArrayElements(env, arg6, lparg6, 0);
-	OS_NATIVE_EXIT(env, that, WideCharToMultiByte__II_3CI_3BI_3B_3Z_FUNC);
+	OS_NATIVE_EXIT(env, that, WideCharToMultiByte__II_3CI_3BI_3B_3I_FUNC);
 	return rc;
 }
 #endif
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.c b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.c
index 98cbc56c2f..00b87eb97a 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.c
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.c
@@ -735,8 +735,8 @@ char * OS_nativeFunctionNames[] = {
 	"WINDOWPOS_1sizeof",
 	"WNDCLASS_1sizeof",
 	"WaitMessage",
-	"WideCharToMultiByte__II_3CIJI_3B_3Z",
-	"WideCharToMultiByte__II_3CI_3BI_3B_3Z",
+	"WideCharToMultiByte__II_3CIJI_3B_3I",
+	"WideCharToMultiByte__II_3CI_3BI_3B_3I",
 	"WindowFromDC",
 	"WindowFromPoint",
 	"wcslen",
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.h b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.h
index 25aaa597fd..0961395a34 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.h
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/library/os_stats.h
@@ -745,8 +745,8 @@ typedef enum {
 	WINDOWPOS_1sizeof_FUNC,
 	WNDCLASS_1sizeof_FUNC,
 	WaitMessage_FUNC,
-	WideCharToMultiByte__II_3CIJI_3B_3Z_FUNC,
-	WideCharToMultiByte__II_3CI_3BI_3B_3Z_FUNC,
+	WideCharToMultiByte__II_3CIJI_3B_3I_FUNC,
+	WideCharToMultiByte__II_3CI_3BI_3B_3I_FUNC,
 	WindowFromDC_FUNC,
 	WindowFromPoint_FUNC,
 	wcslen_FUNC,
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/org/eclipse/swt/internal/win32/OS.java b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/org/eclipse/swt/internal/win32/OS.java
index 75ded24b39..daff77b6ec 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/win32/org/eclipse/swt/internal/win32/OS.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/win32/org/eclipse/swt/internal/win32/OS.java
@@ -4428,14 +4428,14 @@ public static final native boolean WaitMessage ();
  * @param lpDefaultChar cast=(LPCSTR)
  * @param lpUsedDefaultChar cast=(LPBOOL)
  */
-public static final native int WideCharToMultiByte (int CodePage, int dwFlags, char [] lpWideCharStr, int cchWideChar, byte [] lpMultiByteStr, int cchMultiByte, byte [] lpDefaultChar, boolean [] lpUsedDefaultChar);
+public static final native int WideCharToMultiByte (int CodePage, int dwFlags, char [] lpWideCharStr, int cchWideChar, byte [] lpMultiByteStr, int cchMultiByte, byte [] lpDefaultChar, int [] lpUsedDefaultChar);
 /**
  * @param lpWideCharStr cast=(LPCWSTR),flags=no_out critical
  * @param lpMultiByteStr cast=(LPSTR)
  * @param lpDefaultChar cast=(LPCSTR)
  * @param lpUsedDefaultChar cast=(LPBOOL)
  */
-public static final native int WideCharToMultiByte (int CodePage, int dwFlags, char [] lpWideCharStr, int cchWideChar, long lpMultiByteStr, int cchMultiByte, byte [] lpDefaultChar, boolean [] lpUsedDefaultChar);
+public static final native int WideCharToMultiByte (int CodePage, int dwFlags, char [] lpWideCharStr, int cchWideChar, long lpMultiByteStr, int cchMultiByte, byte [] lpDefaultChar, int [] lpUsedDefaultChar);
 /** @param hDC cast=(HDC) */
 public static final native long WindowFromDC (long hDC);
 /** @param lpPoint flags=struct */
diff --git a/bundles/org.eclipse.swt/Eclipse SWT/common/library/swt.h b/bundles/org.eclipse.swt/Eclipse SWT/common/library/swt.h
index 51ee0dea80..fa228e9971 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT/common/library/swt.h
+++ b/bundles/org.eclipse.swt/Eclipse SWT/common/library/swt.h
@@ -68,6 +68,14 @@ extern JavaVM *JVM;
 			if (hm) var = GetProcAddress(hm, #name); \
 			initialized = 1; \
 		}
+/*
+ * Java's boolean[] may not be used to interface to C on Windows, because:
+ * WINAPI 'BOOL'     is 4 bytes
+ * Java's 'jboolean' is 1 byte
+ * Trying to pass Java's boolean[] will pass wrong data on read and corrupt memory on write.
+ * Use java's 'int[]' as a correct interface to 'BOOL'.
+ */
+#define jbooleanArray DONT_USE_jbooleanArray_USE_intArray_INSTEAD
 #else
 #define CALLING_CONVENTION
 #define LOAD_FLAGS RTLD_LAZY
@@ -80,6 +88,14 @@ extern JavaVM *JVM;
 			initialized = 1; \
 	                CHECK_DLERROR \
 		}
+/*
+ * Java's boolean[] may not be used to interface to C on Linux, because:
+ * GLib's 'gboolean' is 4 bytes
+ * Java's 'jboolean' is 1 byte
+ * Trying to pass Java's boolean[] will pass wrong data on read and corrupt memory on write.
+ * Use java's 'int[]' as a correct interface to 'gboolean'.
+ */
+#define jbooleanArray DONT_USE_jbooleanArray_USE_intArray_INSTEAD
 #endif
 
 void throwOutOfMemory(JNIEnv *env);