html/process_project_source_locations.php

<?php
/*******************************************************************************
 * Copyright (c) 2013-2019 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    Kit Lo (IBM) - [402192] Extract project source files from Git repositories for translation
 *    Denis Roy (Eclipse Foundation) - Bug 550544 - Babel server is not ready for PHP 7
 *******************************************************************************/

/*
 * Extract properties or js files from update sites 
 */
$temp_dir = "/tmp/tmp-babel/";
$files = array();
$files_collected = array(array());

header("Content-type: text/plain");
include("global.php");
InitPage("");

$headless = 0;
if (!isset($User)) {
  echo "User not defined - running headless\n";
  $User = getGenieUser();
  $headless = 1;
}

require(dirname(__FILE__) . "/../classes/file/file.class.php");
global $dbh;

$temp_unzips_dir = $temp_dir . "unzips/";
if (is_dir($temp_unzips_dir)) {
  exec("rm -rf $temp_unzips_dir");
}
mkdir($temp_unzips_dir, 0777, TRUE) || die("***ERROR: Cannot create working directory: $temp_unzips_dir\n");

global $addon;
$context = $addon->callHook('context');
if ($context == "live") {
  $rsync_host = "download.eclipse.org::eclipseFullMirror/";
} else {
  $rsync_host = "rsync.osuosl.org::eclipse/";
}

# Get all active update sites
$sql = "SELECT * FROM project_source_locations AS m 
INNER JOIN release_train_projects AS r ON r.project_id = m.project_id AND r.version = m.version 
INNER JOIN release_trains AS t on t.train_id = r.train_id 
WHERE t.is_active = 1";
$rs_maps = mysqli_query($dbh, $sql);
while($update_site = mysqli_fetch_assoc($rs_maps)) {
  $site_url = $update_site['location'];
  $project_id = $update_site['project_id'];
  $version = $update_site['version'];

  # fix WTP version "3.12 (2018-12)"
  $version_dir = str_replace(" ", ".", $version);
  $version_dir = str_replace("(", "", $version_dir);
  $version_dir = str_replace(")", "", $version_dir);

  # Sample dirs:
  # $site_url           http://git.eclipse.org/c/platform/eclipse.platform.git/snapshot/I20130101-0800.zip
  # $temp_snapshots_dir /tmp/tmp-babel/snapshots/eclipse/4.3/
  # $temp_unzips_dir    /tmp/tmp-babel/unzips/eclipse/4.3/
  $temp_snapshots_dir = $temp_dir . "snapshots/" . $project_id . "/" . $version_dir . "/";
  $temp_unzips_dir = $temp_dir . "unzips/" . $project_id . "/" . $version_dir . "/";

  # Collect all files for this project version
  if (!(isset($files_collected[$project_id]) && isset($files_collected[$project_id][$version]))) {
    $files_collected[$project_id][$version] = 1;
    $sql = "SELECT * FROM files WHERE project_id = \"$project_id\" AND version = \"$version\"";
    $rs_files = mysqli_query($dbh, $sql);
    while ($myrow_files = mysqli_fetch_assoc($rs_files)) {
      $file = new File();
      $file->project_id = $myrow_files['project_id'];
      $file->version = $myrow_files['version'];
      $file->name = $myrow_files['name'];
      $file->plugin_id = $myrow_files['plugin_id'];
      $file->file_id = $myrow_files['file_id'];
      $file->is_active = $myrow_files['is_active'];
      $files[$file->file_id] = $file;
    }
  }

  # Collect all plugin exclude patterns for this project version
  $sql = "SELECT pattern FROM plugin_exclude_patterns WHERE project_id = \"$project_id\" AND version = \"$version\"";
  $rs_patterns = mysqli_query($dbh, $sql);
  $patterns = Array();
  # Add default exclude patterns
  $patterns[] = "/^.*\/feature.properties$/";
  $patterns[] = "/^.*\/build.properties$/";
  $patterns[] = "/^.*\/pom.properties$/";
  $patterns[] = "/^.*\.source\/.*$/";
  $patterns[] = "/^.*\.test\/.*$/";
  $patterns[] = "/^.*\.tests\/.*$/";
  $patterns[] = "/^.*\.testing\/.*$/";
  while ($myrow_patterns = mysqli_fetch_assoc($rs_patterns)) {
    $patterns[] = $myrow_patterns['pattern'];
  }

  exec("mkdir -p $temp_snapshots_dir");
  exec("wget $site_url -O ${temp_snapshots_dir}snapshot.zip");

  # Make unzip dir
  mkdir($temp_unzips_dir, 0777, TRUE);
  chdir($temp_unzips_dir);
  exec("unzip -o -q ${temp_snapshots_dir}snapshot.zip");

  # Collect properties file names
  $properties_file_names = array();
  chdir($temp_unzips_dir);
  exec("find . -name *.properties", $properties_file_names);
  sort($properties_file_names);

  # Parse each properties file
  echo "Start processing properties files in project $project_id version $version...\n";
  echo "  Update site location: $site_url\n";
  foreach ($properties_file_names as $properties_file_name) {
	# extract plugin name
    $file_name = $temp_unzips_dir . $properties_file_name;
    $properties_file_name = substr($properties_file_name, strrpos($properties_file_name, "org."));
    $plugin_id = substr($properties_file_name, 0, strpos($properties_file_name, "/"));
    $pos = strpos($properties_file_name, '/');
    if ($pos !== false) {
      $properties_file_name = substr($properties_file_name, $pos);
    }

    # remove optional source dir, e.g. 'src' or 'src_ant'
    $pos = stripos($properties_file_name, '/org/');
    if ($pos !== false) {
      $properties_file_name = substr($properties_file_name, $pos);
    }
    $pos = strripos($properties_file_name, '/com/');
    if ($pos !== false) {
      $properties_file_name = substr($properties_file_name, $pos);
    }

    # get file ID
    $properties_file_name = $plugin_id . $properties_file_name;
	$file_id = File::getFileID($properties_file_name, $project_id, $version);

    # Match plugin exclude list
    $match = false;
    foreach ($patterns as $pattern) {
      if (preg_match($pattern, $properties_file_name)) {
        $match = true;
        break;
      }
    }

    if (!$match) {
      if ($file_id > 0 && array_key_exists($file_id, $files) && $files[$file_id] != null) {
        # Existing file
        $file = $files[$file_id];
        $file->is_active = 1;
        unset($files[$file_id]);
      } else {
        # New file
        $file = new File();
        $file->project_id = $project_id;
        $file->version = $version;
        $file->name = $properties_file_name;
        $file->plugin_id = $plugin_id;
        $file->is_active = 1;
      }
      if (!$file->save()) {
        echo "***ERROR: Cannot save file $file->name\n";
      } else {
        $file_contents = preg_replace("/\r\n?/", "\n", file_get_contents($file_name));
        $file->parseProperties($file_contents);
        echo "  $properties_file_name\n";
      }
    } else {
      echo "  !!! Excluding $properties_file_name\n";
    }
  }
  echo "Done processing " . sizeof($properties_file_names) . " properties files in project $project_id version $version\n\n";

  # Collect js file names
  $js_file_names = array();
  chdir($temp_unzips_dir);
  exec("find . -name *.js | grep nls/root", $js_file_names);
  sort($js_file_names);

  # Parse each js file
  echo "Start processing js files in project $project_id version $version...\n";
  echo "  Update site location: $site_url\n";
  foreach ($js_file_names as $js_file_name) {
    $file_name = $temp_unzips_dir . $js_file_name;
    $js_file_name = substr($js_file_name, strrpos($js_file_name, "org.eclipse."));
    $plugin_id = substr($js_file_name, 0, strpos($js_file_name, "/"));
	$file_id = File::getFileID($js_file_name, $project_id, $version);

    # Match plugin exclude list
    $match = false;
    foreach ($patterns as $pattern) {
      if (preg_match($pattern, $js_file_name)) {
        $match = true;
        break;
      }
    }

    if (!$match) {
      if ($file_id > 0 && $files[$file_id] != null) {
        # Existing file
        $file = $files[$file_id];
        $file->is_active = 1;
        unset($files[$file_id]);
      } else {
        # New file
        $file = new File();
        $file->project_id = $project_id;
        $file->version = $version;
        $file->name = $js_file_name;
        $file->plugin_id = $plugin_id;
        $file->is_active = 1;
      }
      if (!$file->save()) {
        echo "***ERROR: Cannot save file $file->name\n";
      } else {
        $file_contents = preg_replace("/\r\n?/", "\n", file_get_contents($file_name));
        $file->parseJs($file_contents);
        echo "  $js_file_name\n";
      }
    } else {
      echo "  !!! Excluding $js_file_name\n";
    }
  }
  echo "Done processing " . sizeof($js_file_names) . " js files in project $project_id version $version\n\n";

  chdir($temp_dir);
  exec("rm -rf $temp_snapshots_dir");
  exec("rm -rf $temp_unzips_dir");
}

# Deactivate the rest of the files
echo "Start deactivating inactive properties or js files in all projects above...\n";
foreach ($files as $file) {
  if ($file->is_active == 1) {
    $file->is_active = 0;
    if (!$file->save()) {
      echo "***ERROR: Cannot deactivate file $file->name\n";
    }
    echo "  " . $file->name . "\n";
  } else {
    unset($files[$file->file_id]);
  }
}
echo "Done deactivating " . sizeof($files) . " inactive properties or js files in all projects above\n\n";

chdir($temp_dir);
exec("rm -rf snapshots");
exec("rm -rf unzips");

if ($headless) {
  $User = null;
}

echo "Done\n";
?>