Bug 519886 - eclipse CDT failed to create PTY on macOS 10.13 betacdt_9_2

Fixed by getting rid of legacy code for opening a master pty, and
copying the related code from org.eclipse.cdt.core.linux instead.
Only getpt() had to be replaced by posix_openpt(). Compiled on
OSX 10.12 - tested on MacOSX 10.8 and 10.13 beta 7 (64-bit each).

Change-Id: Idade001aed41abafc0d500695a4b51179bb9b738
Signed-off-by: Martin Oberhuber <mober.at+eclipse@gmail.com>
(cherry picked from commit a0ab38124c6e78eeacedf92feb685f7e4d3002d0)

5 files changed, 52 insertions, 57 deletions

diff --git a/core/org.eclipse.cdt.core.macosx/library/openpty.c b/core/org.eclipse.cdt.core.macosx/library/openpty.c
index 3ad49dfaa45..6a505806b21 100644
--- a/core/org.eclipse.cdt.core.macosx/library/openpty.c
+++ b/core/org.eclipse.cdt.core.macosx/library/openpty.c
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2002, 2010 QNX Software Systems and others.
+ * Copyright (c) 2002, 2017 QNX Software Systems and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -7,7 +7,14 @@
  *
  * Contributors:
  *     QNX Software Systems - initial API and implementation
+ *     Wind River Systems, Inc.
+ *     Mikhail Zabaluev (Nokia) - bug 82744
+ *     Corey Ashford (IBM) - bug 272370, bug 272372
+ *     Martin Oberhuber - [519886] align w/ Linux, support OSX 10.13
  *******************************************************************************/
+
+/* _XOPEN_SOURCE is needed to bring in the header for ptsname */
+#define _XOPEN_SOURCE
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/ioctl.h>
@@ -18,7 +25,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <grp.h>
-#include <termios.h>
 
 #include <stdlib.h>
 
@@ -29,6 +35,7 @@
 
 int ptym_open (char *pts_name);
 int ptys_open (int fdm, const char * pts_name);
+void set_noecho(int fd);
 
 int
 openpty(int *amaster, int *aslave, char *name, struct termios *termp, struct winsize *winp)
@@ -43,6 +50,7 @@ openpty(int *amaster, int *aslave, char *name, struct termios *termp, struct win
 		close(*amaster);
 		return -1;
 	}
+
 	if (name)
 		strcpy(name, line);
 #ifndef TCSAFLUSH
@@ -57,80 +65,67 @@ openpty(int *amaster, int *aslave, char *name, struct termios *termp, struct win
 	return 0;
 }
 
+void
+set_noecho(int fd)
+{
+	struct termios stermios;
+	if (tcgetattr(fd, &stermios) < 0) {
+		return ;
+	}
+
+	/* turn off echo */
+	stermios.c_lflag &= ~(ECHO | ECHOE | ECHOK | ECHONL);
+	/* Turn off the NL to CR/NL mapping ou output.  */
+	/*stermios.c_oflag &= ~(ONLCR);*/
+
+	stermios.c_iflag |= (IGNCR);
+
+	tcsetattr(fd, TCSANOW, &stermios);
+}
+
 int
 ptym_open(char * pts_name)
 {
-	char *ptr1, *ptr2;
 	int fdm;
-	
-	strcpy(pts_name, "/dev/ptyXY");
-	/* array index: 012345689 (for references in following code) */
-	for (ptr1 = "pqrstuvwxyzPQRST"; *ptr1 != 0; ptr1++) {
-		pts_name[8] = *ptr1;
-		for (ptr2 = "0123456789abcdef"; *ptr2 != 0; ptr2++) {
-			pts_name[9] = *ptr2;
-			/* try to open master */
-			fdm = open(pts_name, O_RDWR);
-			if (fdm < 0) {
-				if (errno == ENOENT) {/* different from EIO */
-					return -1;  /* out of pty devices */
-				} else {
-					continue;  /* try next pty device */
-				}
-			}
-			pts_name[5] = 't'; /* chage "pty" to "tty" */
-			return fdm;   /* got it, return fd of master */
-		}
+	char *ptr;
+
+	strcpy(pts_name, "/dev/ptmx");
+	fdm = posix_openpt(O_RDWR|O_NOCTTY);
+	if (fdm < 0)
+		return -1;
+	if (grantpt(fdm) < 0) { /* grant access to slave */
+		close(fdm);
+		return -2;
+	}
+	if (unlockpt(fdm) < 0) { /* clear slave's lock flag */
+		close(fdm);
+		return -3;
+	}
+	ptr = ptsname(fdm);
+	if (ptr == NULL) { /* get slave's name */
+		close (fdm);
+		return -4;
 	}
-	return -1; /* out of pty devices */
+	strcpy(pts_name, ptr); /* return name of slave */
+	return fdm;            /* return fd of master */
 }
 
 int
 ptys_open(int fdm, const char * pts_name)
 {
-	int gid, fds;
-	struct group *grptr;
-
-	grptr = getgrnam("tty");
-	if (grptr != NULL) {
-		gid = grptr->gr_gid;
-	} else {
-		gid = -1;  /* group tty is not in the group file */
-	}
-
-	/* following two functions don't work unless we're root */
-	chown(pts_name, getuid(), gid);
-	chmod(pts_name, S_IRUSR | S_IWUSR | S_IWGRP);
+	int fds;
+	/* following should allocate controlling terminal */
 	fds = open(pts_name, O_RDWR);
 	if (fds < 0) {
 		close(fdm);
-		return -1;
+		return -5;
 	}
-	
+
 #if	defined(TIOCSCTTY)
 	/*  TIOCSCTTY is the BSD way to acquire a controlling terminal. */
 	if (ioctl(fds, TIOCSCTTY, (char *)0) < 0) {
 		// ignore error: this is expected in console-mode
 	}
 #endif
-	
 	return fds;
 }
-
-void
-set_noecho(int fd)
-{
-	struct termios stermios;
-	if (tcgetattr(fd, &stermios) < 0) {
-		return ;
-	}
-
-	/* turn off echo */
-	stermios.c_lflag &= ~(ECHO | ECHOE | ECHOK | ECHONL);
-	/* Turn off the NL to CR/NL mapping ou output.  */
-	/*stermios.c_oflag &= ~(ONLCR);*/
-
-	stermios.c_iflag |= (IGNCR);
-
-	tcsetattr(fd, TCSANOW, &stermios);
-}
diff --git a/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libpty.jnilib b/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libpty.jnilib
index 942b4bad8c4..50ba6991d48 100755
--- a/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libpty.jnilib
+++ b/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libpty.jnilib
diff --git a/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libspawner.jnilib b/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libspawner.jnilib
index dffccabbb60..bde561aed62 100755
--- a/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libspawner.jnilib
+++ b/core/org.eclipse.cdt.core.macosx/os/macosx/x86/libspawner.jnilib
diff --git a/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libpty.jnilib b/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libpty.jnilib
index 904d2369246..b18de5e480c 100755
--- a/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libpty.jnilib
+++ b/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libpty.jnilib
diff --git a/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libspawner.jnilib b/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libspawner.jnilib
index 2d41d7d66ec..8d8c80bac6e 100755
--- a/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libspawner.jnilib
+++ b/core/org.eclipse.cdt.core.macosx/os/macosx/x86_64/libspawner.jnilib