Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 1 | <?php |
| 2 | /******************************************************************************* |
droy | 587ad06 | 2019-09-26 14:23:30 -0400 | [diff] [blame] | 3 | * Copyright (c) 2013-2019 IBM Corporation and others. |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 4 | * All rights reserved. This program and the accompanying materials |
| 5 | * are made available under the terms of the Eclipse Public License v1.0 |
| 6 | * which accompanies this distribution, and is available at |
| 7 | * http://www.eclipse.org/legal/epl-v10.html |
| 8 | * |
| 9 | * Contributors: |
| 10 | * Kit Lo (IBM) - [402192] Extract project source files from Git repositories for translation |
droy | 587ad06 | 2019-09-26 14:23:30 -0400 | [diff] [blame] | 11 | * Denis Roy (Eclipse Foundation) - Bug 550544 - Babel server is not ready for PHP 7 |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 12 | *******************************************************************************/ |
| 13 | |
| 14 | /* |
| 15 | * Extract properties or js files from update sites |
| 16 | */ |
| 17 | $temp_dir = "/tmp/tmp-babel/"; |
| 18 | $files = array(); |
| 19 | $files_collected = array(array()); |
| 20 | |
| 21 | header("Content-type: text/plain"); |
| 22 | include("global.php"); |
| 23 | InitPage(""); |
| 24 | |
| 25 | $headless = 0; |
| 26 | if (!isset($User)) { |
| 27 | echo "User not defined - running headless\n"; |
| 28 | $User = getGenieUser(); |
| 29 | $headless = 1; |
| 30 | } |
| 31 | |
| 32 | require(dirname(__FILE__) . "/../classes/file/file.class.php"); |
| 33 | global $dbh; |
| 34 | |
| 35 | $temp_unzips_dir = $temp_dir . "unzips/"; |
| 36 | if (is_dir($temp_unzips_dir)) { |
| 37 | exec("rm -rf $temp_unzips_dir"); |
| 38 | } |
| 39 | mkdir($temp_unzips_dir, 0777, TRUE) || die("***ERROR: Cannot create working directory: $temp_unzips_dir\n"); |
| 40 | |
| 41 | global $addon; |
| 42 | $context = $addon->callHook('context'); |
| 43 | if ($context == "live") { |
| 44 | $rsync_host = "download.eclipse.org::eclipseFullMirror/"; |
| 45 | } else { |
| 46 | $rsync_host = "rsync.osuosl.org::eclipse/"; |
| 47 | } |
| 48 | |
| 49 | # Get all active update sites |
| 50 | $sql = "SELECT * FROM project_source_locations AS m |
| 51 | INNER JOIN release_train_projects AS r ON r.project_id = m.project_id AND r.version = m.version |
| 52 | INNER JOIN release_trains AS t on t.train_id = r.train_id |
| 53 | WHERE t.is_active = 1"; |
kitlo | 2c8d8a9 | 2018-04-19 13:25:09 -0400 | [diff] [blame] | 54 | $rs_maps = mysqli_query($dbh, $sql); |
kitlo | 1d02709 | 2018-04-19 17:44:07 -0400 | [diff] [blame] | 55 | while($update_site = mysqli_fetch_assoc($rs_maps)) { |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 56 | $site_url = $update_site['location']; |
| 57 | $project_id = $update_site['project_id']; |
| 58 | $version = $update_site['version']; |
| 59 | |
kitlo | 7cb1660 | 2019-01-11 00:13:47 -0500 | [diff] [blame] | 60 | # fix WTP version "3.12 (2018-12)" |
kitlo | 79e51b0 | 2019-01-11 03:25:20 -0500 | [diff] [blame] | 61 | $version_dir = str_replace(" ", ".", $version); |
| 62 | $version_dir = str_replace("(", "", $version_dir); |
| 63 | $version_dir = str_replace(")", "", $version_dir); |
kitlo | 7cb1660 | 2019-01-11 00:13:47 -0500 | [diff] [blame] | 64 | |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 65 | # Sample dirs: |
| 66 | # $site_url http://git.eclipse.org/c/platform/eclipse.platform.git/snapshot/I20130101-0800.zip |
| 67 | # $temp_snapshots_dir /tmp/tmp-babel/snapshots/eclipse/4.3/ |
| 68 | # $temp_unzips_dir /tmp/tmp-babel/unzips/eclipse/4.3/ |
kitlo | 9d9d3dc | 2019-01-11 02:23:42 -0500 | [diff] [blame] | 69 | $temp_snapshots_dir = $temp_dir . "snapshots/" . $project_id . "/" . $version_dir . "/"; |
| 70 | $temp_unzips_dir = $temp_dir . "unzips/" . $project_id . "/" . $version_dir . "/"; |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 71 | |
| 72 | # Collect all files for this project version |
| 73 | if (!(isset($files_collected[$project_id]) && isset($files_collected[$project_id][$version]))) { |
| 74 | $files_collected[$project_id][$version] = 1; |
| 75 | $sql = "SELECT * FROM files WHERE project_id = \"$project_id\" AND version = \"$version\""; |
kitlo | 2c8d8a9 | 2018-04-19 13:25:09 -0400 | [diff] [blame] | 76 | $rs_files = mysqli_query($dbh, $sql); |
kitlo | 1d02709 | 2018-04-19 17:44:07 -0400 | [diff] [blame] | 77 | while ($myrow_files = mysqli_fetch_assoc($rs_files)) { |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 78 | $file = new File(); |
| 79 | $file->project_id = $myrow_files['project_id']; |
| 80 | $file->version = $myrow_files['version']; |
| 81 | $file->name = $myrow_files['name']; |
| 82 | $file->plugin_id = $myrow_files['plugin_id']; |
| 83 | $file->file_id = $myrow_files['file_id']; |
| 84 | $file->is_active = $myrow_files['is_active']; |
| 85 | $files[$file->file_id] = $file; |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | # Collect all plugin exclude patterns for this project version |
| 90 | $sql = "SELECT pattern FROM plugin_exclude_patterns WHERE project_id = \"$project_id\" AND version = \"$version\""; |
kitlo | 2c8d8a9 | 2018-04-19 13:25:09 -0400 | [diff] [blame] | 91 | $rs_patterns = mysqli_query($dbh, $sql); |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 92 | $patterns = Array(); |
| 93 | # Add default exclude patterns |
| 94 | $patterns[] = "/^.*\/feature.properties$/"; |
| 95 | $patterns[] = "/^.*\/build.properties$/"; |
| 96 | $patterns[] = "/^.*\/pom.properties$/"; |
| 97 | $patterns[] = "/^.*\.source\/.*$/"; |
| 98 | $patterns[] = "/^.*\.test\/.*$/"; |
| 99 | $patterns[] = "/^.*\.tests\/.*$/"; |
| 100 | $patterns[] = "/^.*\.testing\/.*$/"; |
kitlo | 1d02709 | 2018-04-19 17:44:07 -0400 | [diff] [blame] | 101 | while ($myrow_patterns = mysqli_fetch_assoc($rs_patterns)) { |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 102 | $patterns[] = $myrow_patterns['pattern']; |
| 103 | } |
| 104 | |
| 105 | exec("mkdir -p $temp_snapshots_dir"); |
| 106 | exec("wget $site_url -O ${temp_snapshots_dir}snapshot.zip"); |
| 107 | |
| 108 | # Make unzip dir |
| 109 | mkdir($temp_unzips_dir, 0777, TRUE); |
| 110 | chdir($temp_unzips_dir); |
| 111 | exec("unzip -o -q ${temp_snapshots_dir}snapshot.zip"); |
| 112 | |
| 113 | # Collect properties file names |
| 114 | $properties_file_names = array(); |
| 115 | chdir($temp_unzips_dir); |
| 116 | exec("find . -name *.properties", $properties_file_names); |
| 117 | sort($properties_file_names); |
| 118 | |
| 119 | # Parse each properties file |
| 120 | echo "Start processing properties files in project $project_id version $version...\n"; |
| 121 | echo " Update site location: $site_url\n"; |
| 122 | foreach ($properties_file_names as $properties_file_name) { |
| 123 | # extract plugin name |
| 124 | $file_name = $temp_unzips_dir . $properties_file_name; |
| 125 | $properties_file_name = substr($properties_file_name, strrpos($properties_file_name, "org.")); |
| 126 | $plugin_id = substr($properties_file_name, 0, strpos($properties_file_name, "/")); |
| 127 | $pos = strpos($properties_file_name, '/'); |
| 128 | if ($pos !== false) { |
| 129 | $properties_file_name = substr($properties_file_name, $pos); |
| 130 | } |
| 131 | |
| 132 | # remove optional source dir, e.g. 'src' or 'src_ant' |
| 133 | $pos = stripos($properties_file_name, '/org/'); |
| 134 | if ($pos !== false) { |
| 135 | $properties_file_name = substr($properties_file_name, $pos); |
| 136 | } |
| 137 | $pos = strripos($properties_file_name, '/com/'); |
| 138 | if ($pos !== false) { |
| 139 | $properties_file_name = substr($properties_file_name, $pos); |
| 140 | } |
| 141 | |
| 142 | # get file ID |
| 143 | $properties_file_name = $plugin_id . $properties_file_name; |
| 144 | $file_id = File::getFileID($properties_file_name, $project_id, $version); |
| 145 | |
| 146 | # Match plugin exclude list |
| 147 | $match = false; |
| 148 | foreach ($patterns as $pattern) { |
| 149 | if (preg_match($pattern, $properties_file_name)) { |
| 150 | $match = true; |
| 151 | break; |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | if (!$match) { |
kitlo | 045c126 | 2018-08-04 21:00:43 -0400 | [diff] [blame] | 156 | if ($file_id > 0 && array_key_exists($file_id, $files) && $files[$file_id] != null) { |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 157 | # Existing file |
| 158 | $file = $files[$file_id]; |
| 159 | $file->is_active = 1; |
| 160 | unset($files[$file_id]); |
| 161 | } else { |
| 162 | # New file |
| 163 | $file = new File(); |
| 164 | $file->project_id = $project_id; |
| 165 | $file->version = $version; |
| 166 | $file->name = $properties_file_name; |
| 167 | $file->plugin_id = $plugin_id; |
| 168 | $file->is_active = 1; |
| 169 | } |
| 170 | if (!$file->save()) { |
| 171 | echo "***ERROR: Cannot save file $file->name\n"; |
| 172 | } else { |
droy | 587ad06 | 2019-09-26 14:23:30 -0400 | [diff] [blame] | 173 | $file_contents = preg_replace("/\r\n?/", "\n", file_get_contents($file_name)); |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 174 | $file->parseProperties($file_contents); |
| 175 | echo " $properties_file_name\n"; |
| 176 | } |
| 177 | } else { |
| 178 | echo " !!! Excluding $properties_file_name\n"; |
| 179 | } |
| 180 | } |
| 181 | echo "Done processing " . sizeof($properties_file_names) . " properties files in project $project_id version $version\n\n"; |
| 182 | |
| 183 | # Collect js file names |
| 184 | $js_file_names = array(); |
| 185 | chdir($temp_unzips_dir); |
| 186 | exec("find . -name *.js | grep nls/root", $js_file_names); |
| 187 | sort($js_file_names); |
| 188 | |
| 189 | # Parse each js file |
| 190 | echo "Start processing js files in project $project_id version $version...\n"; |
| 191 | echo " Update site location: $site_url\n"; |
| 192 | foreach ($js_file_names as $js_file_name) { |
| 193 | $file_name = $temp_unzips_dir . $js_file_name; |
| 194 | $js_file_name = substr($js_file_name, strrpos($js_file_name, "org.eclipse.")); |
| 195 | $plugin_id = substr($js_file_name, 0, strpos($js_file_name, "/")); |
| 196 | $file_id = File::getFileID($js_file_name, $project_id, $version); |
| 197 | |
| 198 | # Match plugin exclude list |
| 199 | $match = false; |
| 200 | foreach ($patterns as $pattern) { |
| 201 | if (preg_match($pattern, $js_file_name)) { |
| 202 | $match = true; |
| 203 | break; |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | if (!$match) { |
| 208 | if ($file_id > 0 && $files[$file_id] != null) { |
| 209 | # Existing file |
| 210 | $file = $files[$file_id]; |
| 211 | $file->is_active = 1; |
| 212 | unset($files[$file_id]); |
| 213 | } else { |
| 214 | # New file |
| 215 | $file = new File(); |
| 216 | $file->project_id = $project_id; |
| 217 | $file->version = $version; |
| 218 | $file->name = $js_file_name; |
| 219 | $file->plugin_id = $plugin_id; |
| 220 | $file->is_active = 1; |
| 221 | } |
| 222 | if (!$file->save()) { |
| 223 | echo "***ERROR: Cannot save file $file->name\n"; |
| 224 | } else { |
droy | 587ad06 | 2019-09-26 14:23:30 -0400 | [diff] [blame] | 225 | $file_contents = preg_replace("/\r\n?/", "\n", file_get_contents($file_name)); |
Kit Lo | 3c7ce14 | 2013-03-02 18:22:40 -0500 | [diff] [blame] | 226 | $file->parseJs($file_contents); |
| 227 | echo " $js_file_name\n"; |
| 228 | } |
| 229 | } else { |
| 230 | echo " !!! Excluding $js_file_name\n"; |
| 231 | } |
| 232 | } |
| 233 | echo "Done processing " . sizeof($js_file_names) . " js files in project $project_id version $version\n\n"; |
| 234 | |
| 235 | chdir($temp_dir); |
| 236 | exec("rm -rf $temp_snapshots_dir"); |
| 237 | exec("rm -rf $temp_unzips_dir"); |
| 238 | } |
| 239 | |
| 240 | # Deactivate the rest of the files |
| 241 | echo "Start deactivating inactive properties or js files in all projects above...\n"; |
| 242 | foreach ($files as $file) { |
| 243 | if ($file->is_active == 1) { |
| 244 | $file->is_active = 0; |
| 245 | if (!$file->save()) { |
| 246 | echo "***ERROR: Cannot deactivate file $file->name\n"; |
| 247 | } |
| 248 | echo " " . $file->name . "\n"; |
| 249 | } else { |
| 250 | unset($files[$file->file_id]); |
| 251 | } |
| 252 | } |
| 253 | echo "Done deactivating " . sizeof($files) . " inactive properties or js files in all projects above\n\n"; |
| 254 | |
| 255 | chdir($temp_dir); |
| 256 | exec("rm -rf snapshots"); |
| 257 | exec("rm -rf unzips"); |
| 258 | |
| 259 | if ($headless) { |
| 260 | $User = null; |
| 261 | } |
| 262 | |
| 263 | echo "Done\n"; |
| 264 | ?> |