blob: c70fec9f54729807606b1ae3b518a51ee6176c10 [file] [log] [blame]
Kit Lo3c7ce142013-03-02 18:22:40 -05001<?php
2/*******************************************************************************
droy587ad062019-09-26 14:23:30 -04003 * Copyright (c) 2013-2019 IBM Corporation and others.
Kit Lo3c7ce142013-03-02 18:22:40 -05004 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
8 *
9 * Contributors:
10 * Kit Lo (IBM) - [402192] Extract project source files from Git repositories for translation
droy587ad062019-09-26 14:23:30 -040011 * Denis Roy (Eclipse Foundation) - Bug 550544 - Babel server is not ready for PHP 7
Kit Lo3c7ce142013-03-02 18:22:40 -050012 *******************************************************************************/
13
14/*
15 * Extract properties or js files from update sites
16 */
17$temp_dir = "/tmp/tmp-babel/";
18$files = array();
19$files_collected = array(array());
20
21header("Content-type: text/plain");
22include("global.php");
23InitPage("");
24
25$headless = 0;
26if (!isset($User)) {
27 echo "User not defined - running headless\n";
28 $User = getGenieUser();
29 $headless = 1;
30}
31
32require(dirname(__FILE__) . "/../classes/file/file.class.php");
33global $dbh;
34
35$temp_unzips_dir = $temp_dir . "unzips/";
36if (is_dir($temp_unzips_dir)) {
37 exec("rm -rf $temp_unzips_dir");
38}
39mkdir($temp_unzips_dir, 0777, TRUE) || die("***ERROR: Cannot create working directory: $temp_unzips_dir\n");
40
41global $addon;
42$context = $addon->callHook('context');
43if ($context == "live") {
44 $rsync_host = "download.eclipse.org::eclipseFullMirror/";
45} else {
46 $rsync_host = "rsync.osuosl.org::eclipse/";
47}
48
49# Get all active update sites
50$sql = "SELECT * FROM project_source_locations AS m
51INNER JOIN release_train_projects AS r ON r.project_id = m.project_id AND r.version = m.version
52INNER JOIN release_trains AS t on t.train_id = r.train_id
53WHERE t.is_active = 1";
kitlo2c8d8a92018-04-19 13:25:09 -040054$rs_maps = mysqli_query($dbh, $sql);
kitlo1d027092018-04-19 17:44:07 -040055while($update_site = mysqli_fetch_assoc($rs_maps)) {
Kit Lo3c7ce142013-03-02 18:22:40 -050056 $site_url = $update_site['location'];
57 $project_id = $update_site['project_id'];
58 $version = $update_site['version'];
59
kitlo7cb16602019-01-11 00:13:47 -050060 # fix WTP version "3.12 (2018-12)"
kitlo79e51b02019-01-11 03:25:20 -050061 $version_dir = str_replace(" ", ".", $version);
62 $version_dir = str_replace("(", "", $version_dir);
63 $version_dir = str_replace(")", "", $version_dir);
kitlo7cb16602019-01-11 00:13:47 -050064
Kit Lo3c7ce142013-03-02 18:22:40 -050065 # Sample dirs:
66 # $site_url http://git.eclipse.org/c/platform/eclipse.platform.git/snapshot/I20130101-0800.zip
67 # $temp_snapshots_dir /tmp/tmp-babel/snapshots/eclipse/4.3/
68 # $temp_unzips_dir /tmp/tmp-babel/unzips/eclipse/4.3/
kitlo9d9d3dc2019-01-11 02:23:42 -050069 $temp_snapshots_dir = $temp_dir . "snapshots/" . $project_id . "/" . $version_dir . "/";
70 $temp_unzips_dir = $temp_dir . "unzips/" . $project_id . "/" . $version_dir . "/";
Kit Lo3c7ce142013-03-02 18:22:40 -050071
72 # Collect all files for this project version
73 if (!(isset($files_collected[$project_id]) && isset($files_collected[$project_id][$version]))) {
74 $files_collected[$project_id][$version] = 1;
75 $sql = "SELECT * FROM files WHERE project_id = \"$project_id\" AND version = \"$version\"";
kitlo2c8d8a92018-04-19 13:25:09 -040076 $rs_files = mysqli_query($dbh, $sql);
kitlo1d027092018-04-19 17:44:07 -040077 while ($myrow_files = mysqli_fetch_assoc($rs_files)) {
Kit Lo3c7ce142013-03-02 18:22:40 -050078 $file = new File();
79 $file->project_id = $myrow_files['project_id'];
80 $file->version = $myrow_files['version'];
81 $file->name = $myrow_files['name'];
82 $file->plugin_id = $myrow_files['plugin_id'];
83 $file->file_id = $myrow_files['file_id'];
84 $file->is_active = $myrow_files['is_active'];
85 $files[$file->file_id] = $file;
86 }
87 }
88
89 # Collect all plugin exclude patterns for this project version
90 $sql = "SELECT pattern FROM plugin_exclude_patterns WHERE project_id = \"$project_id\" AND version = \"$version\"";
kitlo2c8d8a92018-04-19 13:25:09 -040091 $rs_patterns = mysqli_query($dbh, $sql);
Kit Lo3c7ce142013-03-02 18:22:40 -050092 $patterns = Array();
93 # Add default exclude patterns
94 $patterns[] = "/^.*\/feature.properties$/";
95 $patterns[] = "/^.*\/build.properties$/";
96 $patterns[] = "/^.*\/pom.properties$/";
97 $patterns[] = "/^.*\.source\/.*$/";
98 $patterns[] = "/^.*\.test\/.*$/";
99 $patterns[] = "/^.*\.tests\/.*$/";
100 $patterns[] = "/^.*\.testing\/.*$/";
kitlo1d027092018-04-19 17:44:07 -0400101 while ($myrow_patterns = mysqli_fetch_assoc($rs_patterns)) {
Kit Lo3c7ce142013-03-02 18:22:40 -0500102 $patterns[] = $myrow_patterns['pattern'];
103 }
104
105 exec("mkdir -p $temp_snapshots_dir");
106 exec("wget $site_url -O ${temp_snapshots_dir}snapshot.zip");
107
108 # Make unzip dir
109 mkdir($temp_unzips_dir, 0777, TRUE);
110 chdir($temp_unzips_dir);
111 exec("unzip -o -q ${temp_snapshots_dir}snapshot.zip");
112
113 # Collect properties file names
114 $properties_file_names = array();
115 chdir($temp_unzips_dir);
116 exec("find . -name *.properties", $properties_file_names);
117 sort($properties_file_names);
118
119 # Parse each properties file
120 echo "Start processing properties files in project $project_id version $version...\n";
121 echo " Update site location: $site_url\n";
122 foreach ($properties_file_names as $properties_file_name) {
123 # extract plugin name
124 $file_name = $temp_unzips_dir . $properties_file_name;
125 $properties_file_name = substr($properties_file_name, strrpos($properties_file_name, "org."));
126 $plugin_id = substr($properties_file_name, 0, strpos($properties_file_name, "/"));
127 $pos = strpos($properties_file_name, '/');
128 if ($pos !== false) {
129 $properties_file_name = substr($properties_file_name, $pos);
130 }
131
132 # remove optional source dir, e.g. 'src' or 'src_ant'
133 $pos = stripos($properties_file_name, '/org/');
134 if ($pos !== false) {
135 $properties_file_name = substr($properties_file_name, $pos);
136 }
137 $pos = strripos($properties_file_name, '/com/');
138 if ($pos !== false) {
139 $properties_file_name = substr($properties_file_name, $pos);
140 }
141
142 # get file ID
143 $properties_file_name = $plugin_id . $properties_file_name;
144 $file_id = File::getFileID($properties_file_name, $project_id, $version);
145
146 # Match plugin exclude list
147 $match = false;
148 foreach ($patterns as $pattern) {
149 if (preg_match($pattern, $properties_file_name)) {
150 $match = true;
151 break;
152 }
153 }
154
155 if (!$match) {
kitlo045c1262018-08-04 21:00:43 -0400156 if ($file_id > 0 && array_key_exists($file_id, $files) && $files[$file_id] != null) {
Kit Lo3c7ce142013-03-02 18:22:40 -0500157 # Existing file
158 $file = $files[$file_id];
159 $file->is_active = 1;
160 unset($files[$file_id]);
161 } else {
162 # New file
163 $file = new File();
164 $file->project_id = $project_id;
165 $file->version = $version;
166 $file->name = $properties_file_name;
167 $file->plugin_id = $plugin_id;
168 $file->is_active = 1;
169 }
170 if (!$file->save()) {
171 echo "***ERROR: Cannot save file $file->name\n";
172 } else {
droy587ad062019-09-26 14:23:30 -0400173 $file_contents = preg_replace("/\r\n?/", "\n", file_get_contents($file_name));
Kit Lo3c7ce142013-03-02 18:22:40 -0500174 $file->parseProperties($file_contents);
175 echo " $properties_file_name\n";
176 }
177 } else {
178 echo " !!! Excluding $properties_file_name\n";
179 }
180 }
181 echo "Done processing " . sizeof($properties_file_names) . " properties files in project $project_id version $version\n\n";
182
183 # Collect js file names
184 $js_file_names = array();
185 chdir($temp_unzips_dir);
186 exec("find . -name *.js | grep nls/root", $js_file_names);
187 sort($js_file_names);
188
189 # Parse each js file
190 echo "Start processing js files in project $project_id version $version...\n";
191 echo " Update site location: $site_url\n";
192 foreach ($js_file_names as $js_file_name) {
193 $file_name = $temp_unzips_dir . $js_file_name;
194 $js_file_name = substr($js_file_name, strrpos($js_file_name, "org.eclipse."));
195 $plugin_id = substr($js_file_name, 0, strpos($js_file_name, "/"));
196 $file_id = File::getFileID($js_file_name, $project_id, $version);
197
198 # Match plugin exclude list
199 $match = false;
200 foreach ($patterns as $pattern) {
201 if (preg_match($pattern, $js_file_name)) {
202 $match = true;
203 break;
204 }
205 }
206
207 if (!$match) {
208 if ($file_id > 0 && $files[$file_id] != null) {
209 # Existing file
210 $file = $files[$file_id];
211 $file->is_active = 1;
212 unset($files[$file_id]);
213 } else {
214 # New file
215 $file = new File();
216 $file->project_id = $project_id;
217 $file->version = $version;
218 $file->name = $js_file_name;
219 $file->plugin_id = $plugin_id;
220 $file->is_active = 1;
221 }
222 if (!$file->save()) {
223 echo "***ERROR: Cannot save file $file->name\n";
224 } else {
droy587ad062019-09-26 14:23:30 -0400225 $file_contents = preg_replace("/\r\n?/", "\n", file_get_contents($file_name));
Kit Lo3c7ce142013-03-02 18:22:40 -0500226 $file->parseJs($file_contents);
227 echo " $js_file_name\n";
228 }
229 } else {
230 echo " !!! Excluding $js_file_name\n";
231 }
232 }
233 echo "Done processing " . sizeof($js_file_names) . " js files in project $project_id version $version\n\n";
234
235 chdir($temp_dir);
236 exec("rm -rf $temp_snapshots_dir");
237 exec("rm -rf $temp_unzips_dir");
238}
239
240# Deactivate the rest of the files
241echo "Start deactivating inactive properties or js files in all projects above...\n";
242foreach ($files as $file) {
243 if ($file->is_active == 1) {
244 $file->is_active = 0;
245 if (!$file->save()) {
246 echo "***ERROR: Cannot deactivate file $file->name\n";
247 }
248 echo " " . $file->name . "\n";
249 } else {
250 unset($files[$file->file_id]);
251 }
252}
253echo "Done deactivating " . sizeof($files) . " inactive properties or js files in all projects above\n\n";
254
255chdir($temp_dir);
256exec("rm -rf snapshots");
257exec("rm -rf unzips");
258
259if ($headless) {
260 $User = null;
261}
262
263echo "Done\n";
264?>