From 73e0fc13cf4ae053658a84d5237afc819828c030 Mon Sep 17 00:00:00 2001 From: jmisinco Date: Fri, 25 Apr 2014 12:09:22 -0700 Subject: feature[ats_ATS40055]: Create tmz file importer Change-Id: I6c3c1cb8a4fb9950f8d08c96af8b4a4cb7ef27b3 --- .../disposition/rest/internal/DispoApiImpl.java | 10 +- .../internal/importer/AbstractDispoImporter.java | 169 +++++++++++++ .../rest/internal/importer/DispoImporter.java | 269 --------------------- .../internal/importer/DispoImporterFactory.java | 49 ++++ .../rest/internal/importer/TmoImporter.java | 145 +++++++++++ .../rest/internal/importer/TmzImporter.java | 176 ++++++++++++++ 6 files changed, 547 insertions(+), 271 deletions(-) create mode 100644 plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/AbstractDispoImporter.java delete mode 100644 plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporter.java create mode 100644 plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporterFactory.java create mode 100644 plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmoImporter.java create mode 100644 plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmzImporter.java (limited to 'plugins/org.eclipse.osee.disposition.rest') diff --git a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/DispoApiImpl.java b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/DispoApiImpl.java index 9d1f3417b80..c0d2f997f16 100644 --- a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/DispoApiImpl.java +++ b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/DispoApiImpl.java @@ -10,6 +10,7 @@ *******************************************************************************/ package org.eclipse.osee.disposition.rest.internal; +import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -25,7 +26,9 @@ import org.eclipse.osee.disposition.model.DispoSetDescriptorData; import org.eclipse.osee.disposition.model.DispoStrings; import org.eclipse.osee.disposition.model.Note; import org.eclipse.osee.disposition.rest.DispoApi; -import org.eclipse.osee.disposition.rest.internal.importer.DispoImporter; +import org.eclipse.osee.disposition.rest.internal.importer.AbstractDispoImporter; +import org.eclipse.osee.disposition.rest.internal.importer.DispoImporterFactory; +import org.eclipse.osee.disposition.rest.internal.importer.DispoImporterFactory.ImportFormat; import org.eclipse.osee.disposition.rest.util.DispoFactory; import org.eclipse.osee.disposition.rest.util.DispoUtil; import org.eclipse.osee.executor.admin.ExecutorAdmin; @@ -53,6 +56,7 @@ public class DispoApiImpl implements DispoApi { private DispoConnector dispoConnector; private DispoFactory dispoFactory; private DispoResolutionValidator resolutionValidator; + private DispoImporterFactory importerFactory; public void setExecutor(ExecutorAdmin executor) { this.executor = executor; @@ -81,6 +85,7 @@ public class DispoApiImpl implements DispoApi { public void start() { logger.trace("Starting DispoApiImpl..."); dispoFactory = new DispoFactoryImpl(); + importerFactory = new DispoImporterFactory(dataFactory, executor, logger); } public void stop() { @@ -371,8 +376,9 @@ public class DispoApiImpl implements DispoApi { if (operation.equals(DispoStrings.Operation_Import)) { try { HashMap nameToItemMap = getItemsMap(program, setToEdit); + AbstractDispoImporter importer = importerFactory.createImporter(ImportFormat.TMO); List itemsFromParse = - DispoImporter.importDirectory(nameToItemMap, setToEdit.getImportPath(), dataFactory, executor); + importer.importDirectory(nameToItemMap, new File(setToEdit.getImportPath())); List itemsToCreate = new ArrayList(); List itemsToEdit = new ArrayList(); diff --git a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/AbstractDispoImporter.java b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/AbstractDispoImporter.java new file mode 100644 index 00000000000..83d43c896d6 --- /dev/null +++ b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/AbstractDispoImporter.java @@ -0,0 +1,169 @@ +/******************************************************************************* + * Copyright (c) 2014 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.disposition.rest.internal.importer; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.eclipse.osee.disposition.model.Discrepancy; +import org.eclipse.osee.disposition.model.DispoAnnotationData; +import org.eclipse.osee.disposition.model.DispoItem; +import org.eclipse.osee.disposition.model.DispoItemData; +import org.eclipse.osee.disposition.model.DispoStrings; +import org.eclipse.osee.disposition.rest.internal.LocationRangesCompressor; +import org.eclipse.osee.disposition.rest.util.DispoUtil; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * @author John Misinco + * @author Angel Avila + */ +public abstract class AbstractDispoImporter { + + public abstract List importDirectory(Map exisitingItems, File filesDir); + + protected void mergeDispoItems(DispoItemData newItem, DispoItem oldItem) throws JSONException { + newItem.setCreationDate(oldItem.getCreationDate()); + JSONObject newItemDiscrepancies = newItem.getDiscrepanciesList(); + JSONArray oldAnnotations = oldItem.getAnnotationsList(); + HashMap idsToUpdate = + matchupOldDiscrepancies(oldItem.getDiscrepanciesList(), newItemDiscrepancies, oldAnnotations); + updateTestPointNumbersForAnntations(idsToUpdate, oldAnnotations, newItemDiscrepancies); + newItem.setGuid(oldItem.getGuid()); + newItem.setAnnotationsList(oldAnnotations); + } + + private void updateTestPointNumbersForAnntations(HashMap idsToUpdate, JSONArray annotations, JSONObject discrepancies) throws JSONException { + for (int j = 0; j < annotations.length(); j++) { + JSONObject annotationAsJson = annotations.getJSONObject(j); + DispoAnnotationData annotation = DispoUtil.jsonObjToDispoAnnotationData(annotationAsJson); + JSONArray idsOfCoveredDiscrepancies = annotation.getIdsOfCoveredDiscrepancies(); + + int length = idsOfCoveredDiscrepancies.length(); + for (int i = 0; i < length; i++) { + String coveredId = idsOfCoveredDiscrepancies.getString(i); + if (idsToUpdate.containsKey(coveredId)) { + String newLocRef = rebuildLocRef(idsOfCoveredDiscrepancies, discrepancies, idsToUpdate); + annotation.setLocationRefs(newLocRef); + JSONObject updatedAnnotationAsJson = DispoUtil.annotationToJsonObj(annotation); + annotations.put(annotation.getIndex(), updatedAnnotationAsJson); + break; // We can break here because we're gonna reconstruct the whole loc refs for the annotation, no need to keep checking + } + } + } + } + + private String rebuildLocRef(JSONArray idsOfCoveredDiscrepancies, JSONObject discrepancies, HashMap idsToUpdate) throws JSONException { + int length = idsOfCoveredDiscrepancies.length(); + List testPointNumber = new ArrayList(); + for (int i = 0; i < length; i++) { + String id = idsOfCoveredDiscrepancies.getString(i); + if (discrepancies.has(id)) { + JSONObject discrepancyAsObject = discrepancies.getJSONObject(id); + Discrepancy discrepancy = DispoUtil.jsonObjToDiscrepancy(discrepancyAsObject); + testPointNumber.add(discrepancy.getLocation()); + } else { + String justTestPoint = id.replaceAll(DispoStrings.DeletedDiscrepancy, ""); + testPointNumber.add(Integer.valueOf(justTestPoint)); + } + + } + + Collections.sort(testPointNumber); + + return LocationRangesCompressor.compress(testPointNumber); + } + + @SuppressWarnings("unchecked") + private HashMap matchupOldDiscrepancies(JSONObject oldDiscrepancies, JSONObject newDiscrepancies, JSONArray annotations) throws JSONException { + HashMap textToNewDiscrepancies = createMap(newDiscrepancies); + HashMap idsToUpdate = new HashMap(); + + Iterator iterator = oldDiscrepancies.keys(); + while (iterator.hasNext()) { + String key = iterator.next(); + JSONObject oldDiscrepancyAsJson = oldDiscrepancies.getJSONObject(key); + Discrepancy oldDiscrepany = DispoUtil.jsonObjToDiscrepancy(oldDiscrepancyAsJson); + String normalizedText = oldDiscrepany.getText().replaceFirst(".*?\\.", ""); + + Discrepancy matchedNewDiscrepancy = textToNewDiscrepancies.get(normalizedText); + int oldTestPointNumber = oldDiscrepany.getLocation(); + if (matchedNewDiscrepancy != null) { + // Transfer the id from the old discrepancy to the new one + String idToReplace = matchedNewDiscrepancy.getId(); + String idOfOldDiscrep = oldDiscrepany.getId(); + matchedNewDiscrepancy.setId(idOfOldDiscrep); + + int newTestPointNumber = matchedNewDiscrepancy.getLocation(); + + if (oldTestPointNumber != newTestPointNumber) { + idsToUpdate.put(idOfOldDiscrep, newTestPointNumber); + } + + JSONObject matchedNewDiscrepAsJson = DispoUtil.discrepancyToJsonObj(matchedNewDiscrepancy); + newDiscrepancies.remove(idToReplace); + newDiscrepancies.put(idOfOldDiscrep, matchedNewDiscrepAsJson); + } else { + int outdateNumber = oldTestPointNumber * -1; + idsToUpdate.put(DispoStrings.DeletedDiscrepancy + outdateNumber, outdateNumber); + removeDiscrepancyFromAnnotation(oldDiscrepany, annotations); + } + } + return idsToUpdate; + } + + @SuppressWarnings("unchecked") + private HashMap createMap(JSONObject discrepancies) throws JSONException { + HashMap textToDiscrepancy = new HashMap(); + Iterator iterator = discrepancies.keys(); + while (iterator.hasNext()) { + String key = iterator.next(); + JSONObject discrepancyAsObject = discrepancies.getJSONObject(key); + Discrepancy discrepancy = DispoUtil.jsonObjToDiscrepancy(discrepancyAsObject); + String normalizedText = discrepancy.getText().replaceFirst(".*?\\.", ""); // Want to exclude Point number from text we match with + textToDiscrepancy.put(normalizedText, discrepancy); + } + return textToDiscrepancy; + } + + private void removeDiscrepancyFromAnnotation(Discrepancy toRemove, JSONArray annotations) throws JSONException { + for (int i = 0; i < annotations.length(); i++) { + JSONObject annotationAsJson = annotations.getJSONObject(i); + DispoAnnotationData annotation = DispoUtil.jsonObjToDispoAnnotationData(annotationAsJson); + JSONArray idsOfCoveredDiscrepancies = annotation.getIdsOfCoveredDiscrepancies(); + if (idsOfCoveredDiscrepancies.toString().contains(toRemove.getId())) { + replaceIdInList(toRemove, idsOfCoveredDiscrepancies); + annotation.setIsConnected(false); + } + JSONObject updatedAnnotationAsJson = DispoUtil.annotationToJsonObj(annotation); + annotations.put(annotation.getIndex(), updatedAnnotationAsJson); + } + } + + private void replaceIdInList(Discrepancy discrepany, JSONArray idsList) throws JSONException { + int length = idsList.length(); + String id = discrepany.getId(); + for (int i = 0; i < length; i++) { + if (id.equals(idsList.getString(i))) { + int testPoint = discrepany.getLocation() * -1; + String newMockId = DispoStrings.DeletedDiscrepancy + testPoint; + idsList.put(i, newMockId); + break; + } + } + } +} diff --git a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporter.java b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporter.java deleted file mode 100644 index b0576278aef..00000000000 --- a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporter.java +++ /dev/null @@ -1,269 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2013 Boeing. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * Boeing - initial API and implementation - *******************************************************************************/ -package org.eclipse.osee.disposition.rest.internal.importer; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FilenameFilter; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; -import org.eclipse.osee.disposition.model.Discrepancy; -import org.eclipse.osee.disposition.model.DispoAnnotationData; -import org.eclipse.osee.disposition.model.DispoItem; -import org.eclipse.osee.disposition.model.DispoItemData; -import org.eclipse.osee.disposition.model.DispoStrings; -import org.eclipse.osee.disposition.rest.internal.DispoDataFactory; -import org.eclipse.osee.disposition.rest.internal.LocationRangesCompressor; -import org.eclipse.osee.disposition.rest.util.DispoUtil; -import org.eclipse.osee.executor.admin.ExecutorAdmin; -import org.eclipse.osee.framework.jdk.core.util.Lib; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -/** - * @author Angel Avila - */ -public class DispoImporter { - - public DispoImporter() { - - } - - public static List importDirectory(HashMap exisitingItems, String path, final DispoDataFactory dataFactory, ExecutorAdmin executor) throws Exception { - List toReturn = new LinkedList(); - File tmoDirectory = new File(path); - if (tmoDirectory.isDirectory()) { - TmoFileFilter filter = new TmoFileFilter(); - File[] files = tmoDirectory.listFiles(filter); - List listOfFiles = Arrays.asList(files); - int numThreads = 8; - int partitionSize = listOfFiles.size() / numThreads; - - int remainder = listOfFiles.size() % numThreads; - int startIndex = 0; - int endIndex = 0; - List>> futures = new LinkedList>>(); - for (int i = 0; i < numThreads; i++) { - startIndex = endIndex; - endIndex = startIndex + partitionSize; - if (i == 0) { - endIndex += remainder; - } - List sublist = listOfFiles.subList(startIndex, endIndex); - Worker worker = new Worker(sublist, dataFactory, exisitingItems); - Future> future = executor.schedule(worker); - futures.add(future); - } - for (Future> future : futures) { - toReturn.addAll(future.get()); - } - } - return toReturn; - } - - private static final class Worker implements Callable> { - - private final List sublist; - private final DispoDataFactory dataFactory; - HashMap exisitingItems; - - public Worker(List sublist, DispoDataFactory dataFactory, HashMap exisitingItems) { - super(); - this.sublist = sublist; - this.dataFactory = dataFactory; - this.exisitingItems = exisitingItems; - } - - @Override - public List call() throws Exception { - List fromThread = new LinkedList(); - for (File file : sublist) { - InputStream inputStream = null; - try { - inputStream = new FileInputStream(file); - - String scriptName = file.getName().replaceAll("\\..*", ""); - - DispoItemData itemToBuild = new DispoItemData(); - // We already have an item with this name so we now have to check the dates - if (exisitingItems.containsKey(scriptName)) { - DispoItem oldItem = exisitingItems.get(scriptName); - Date lastUpdate = oldItem.getLastUpdate(); - boolean wasSameFile = - DiscrepancyParser.buildItemFromFile(itemToBuild, file.getName(), inputStream, false, lastUpdate); - if (!wasSameFile) { - itemToBuild.setCreationDate(oldItem.getCreationDate()); - JSONObject newItemDiscrepancies = itemToBuild.getDiscrepanciesList(); - JSONArray oldAnnotations = oldItem.getAnnotationsList(); - HashMap idsToUpdate = - matchupOldDiscrepancies(oldItem.getDiscrepanciesList(), newItemDiscrepancies, oldAnnotations); - updateTestPointNumbersForAnntations(idsToUpdate, oldAnnotations, newItemDiscrepancies); - itemToBuild.setGuid(oldItem.getGuid()); - itemToBuild.setAnnotationsList(oldAnnotations); - fromThread.add(itemToBuild); - } - } else { - DiscrepancyParser.buildItemFromFile(itemToBuild, file.getName(), inputStream, true, new Date()); - dataFactory.initDispoItem(itemToBuild); - fromThread.add(itemToBuild); - } - } finally { - Lib.close(inputStream); - } - } - return fromThread; - } - }; - - private static void updateTestPointNumbersForAnntations(HashMap idsToUpdate, JSONArray annotations, JSONObject discrepancies) throws JSONException { - for (int j = 0; j < annotations.length(); j++) { - JSONObject annotationAsJson = annotations.getJSONObject(j); - DispoAnnotationData annotation = DispoUtil.jsonObjToDispoAnnotationData(annotationAsJson); - JSONArray idsOfCoveredDiscrepancies = annotation.getIdsOfCoveredDiscrepancies(); - - int length = idsOfCoveredDiscrepancies.length(); - for (int i = 0; i < length; i++) { - String coveredId = idsOfCoveredDiscrepancies.getString(i); - if (idsToUpdate.containsKey(coveredId)) { - String newLocRef = rebuildLocRef(idsOfCoveredDiscrepancies, discrepancies, idsToUpdate); - annotation.setLocationRefs(newLocRef); - JSONObject updatedAnnotationAsJson = DispoUtil.annotationToJsonObj(annotation); - annotations.put(annotation.getIndex(), updatedAnnotationAsJson); - break; // We can break here because we're gonna reconstruct the whole loc refs for the annotation, no need to keep checking - } - } - } - } - - private static String rebuildLocRef(JSONArray idsOfCoveredDiscrepancies, JSONObject discrepancies, HashMap idsToUpdate) throws JSONException { - int length = idsOfCoveredDiscrepancies.length(); - List testPointNumber = new ArrayList(); - for (int i = 0; i < length; i++) { - String id = idsOfCoveredDiscrepancies.getString(i); - if (discrepancies.has(id)) { - JSONObject discrepancyAsObject = discrepancies.getJSONObject(id); - Discrepancy discrepancy = DispoUtil.jsonObjToDiscrepancy(discrepancyAsObject); - testPointNumber.add(discrepancy.getLocation()); - } else { - String justTestPoint = id.replaceAll(DispoStrings.DeletedDiscrepancy, ""); - testPointNumber.add(Integer.valueOf(justTestPoint)); - } - - } - - Collections.sort(testPointNumber); - - return LocationRangesCompressor.compress(testPointNumber); - } - - @SuppressWarnings("unchecked") - private static HashMap matchupOldDiscrepancies(JSONObject oldDiscrepancies, JSONObject newDiscrepancies, JSONArray annotations) throws JSONException { - HashMap textToNewDiscrepancies = createMap(newDiscrepancies); - HashMap idsToUpdate = new HashMap(); - - Iterator iterator = oldDiscrepancies.keys(); - while (iterator.hasNext()) { - String key = iterator.next(); - JSONObject oldDiscrepancyAsJson = oldDiscrepancies.getJSONObject(key); - Discrepancy oldDiscrepany = DispoUtil.jsonObjToDiscrepancy(oldDiscrepancyAsJson); - String normalizedText = oldDiscrepany.getText().replaceFirst(".*?\\.", ""); - - Discrepancy matchedNewDiscrepancy = textToNewDiscrepancies.get(normalizedText); - int oldTestPointNumber = oldDiscrepany.getLocation(); - if (matchedNewDiscrepancy != null) { - // Transfer the id from the old discrepancy to the new one - String idToReplace = matchedNewDiscrepancy.getId(); - String idOfOldDiscrep = oldDiscrepany.getId(); - matchedNewDiscrepancy.setId(idOfOldDiscrep); - - int newTestPointNumber = matchedNewDiscrepancy.getLocation(); - - if (oldTestPointNumber != newTestPointNumber) { - idsToUpdate.put(idOfOldDiscrep, newTestPointNumber); - } - - JSONObject matchedNewDiscrepAsJson = DispoUtil.discrepancyToJsonObj(matchedNewDiscrepancy); - newDiscrepancies.remove(idToReplace); - newDiscrepancies.put(idOfOldDiscrep, matchedNewDiscrepAsJson); - } else { - int outdateNumber = oldTestPointNumber * -1; - idsToUpdate.put(DispoStrings.DeletedDiscrepancy + outdateNumber, outdateNumber); - removeDiscrepancyFromAnnotation(oldDiscrepany, annotations); - - } - } - - return idsToUpdate; - - } - - private static void removeDiscrepancyFromAnnotation(Discrepancy toRemove, JSONArray annotations) throws JSONException { - for (int i = 0; i < annotations.length(); i++) { - JSONObject annotationAsJson = annotations.getJSONObject(i); - DispoAnnotationData annotation = DispoUtil.jsonObjToDispoAnnotationData(annotationAsJson); - JSONArray idsOfCoveredDiscrepancies = annotation.getIdsOfCoveredDiscrepancies(); - if (idsOfCoveredDiscrepancies.toString().contains(toRemove.getId())) { - replaceIdInList(toRemove, idsOfCoveredDiscrepancies); - annotation.setIsConnected(false); - } - - JSONObject updatedAnnotationAsJson = DispoUtil.annotationToJsonObj(annotation); - annotations.put(annotation.getIndex(), updatedAnnotationAsJson); - } - } - - private static void replaceIdInList(Discrepancy discrepany, JSONArray idsList) throws JSONException { - int length = idsList.length(); - String id = discrepany.getId(); - for (int i = 0; i < length; i++) { - if (id.equals(idsList.getString(i))) { - int testPoint = discrepany.getLocation() * -1; - String newMockId = DispoStrings.DeletedDiscrepancy + testPoint; - idsList.put(i, newMockId); - break; - } - } - } - - @SuppressWarnings("unchecked") - private static HashMap createMap(JSONObject discrepancies) throws JSONException { - HashMap textToDiscrepancy = new HashMap(); - Iterator iterator = discrepancies.keys(); - while (iterator.hasNext()) { - String key = iterator.next(); - JSONObject discrepancyAsObject = discrepancies.getJSONObject(key); - Discrepancy discrepancy = DispoUtil.jsonObjToDiscrepancy(discrepancyAsObject); - String normalizedText = discrepancy.getText().replaceFirst(".*?\\.", ""); // Want to exclude Point number from text we match with - textToDiscrepancy.put(normalizedText, discrepancy); - } - - return textToDiscrepancy; - } - - private static final class TmoFileFilter implements FilenameFilter { - - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".tmo"); - } - } - -} diff --git a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporterFactory.java b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporterFactory.java new file mode 100644 index 00000000000..9dce9e2b4bf --- /dev/null +++ b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/DispoImporterFactory.java @@ -0,0 +1,49 @@ +/******************************************************************************* + * Copyright (c) 2014 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.disposition.rest.internal.importer; + +import org.eclipse.osee.disposition.rest.internal.DispoDataFactory; +import org.eclipse.osee.executor.admin.ExecutorAdmin; +import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; +import org.eclipse.osee.logger.Log; + +/** + * @author John Misinco + */ +public class DispoImporterFactory { + + private final DispoDataFactory dataFactory; + private final ExecutorAdmin executor; + private final Log logger; + + public enum ImportFormat { + TMO, + TMZ + }; + + public DispoImporterFactory(DispoDataFactory dataFactory, ExecutorAdmin executor, Log logger) { + this.dataFactory = dataFactory; + this.executor = executor; + this.logger = logger; + } + + public AbstractDispoImporter createImporter(ImportFormat format) { + switch (format) { + case TMO: + return new TmoImporter(dataFactory, executor, logger); + case TMZ: + return new TmzImporter(logger, dataFactory); + default: + throw new OseeArgumentException("Unsupported format type: [%s]", format); + } + } + +} diff --git a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmoImporter.java b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmoImporter.java new file mode 100644 index 00000000000..1f1fca2d4bb --- /dev/null +++ b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmoImporter.java @@ -0,0 +1,145 @@ +/******************************************************************************* + * Copyright (c) 2013 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.disposition.rest.internal.importer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FilenameFilter; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import org.eclipse.osee.disposition.model.DispoItem; +import org.eclipse.osee.disposition.model.DispoItemData; +import org.eclipse.osee.disposition.rest.internal.DispoDataFactory; +import org.eclipse.osee.executor.admin.ExecutorAdmin; +import org.eclipse.osee.framework.jdk.core.util.Lib; +import org.eclipse.osee.logger.Log; + +/** + * @author Angel Avila + */ +public class TmoImporter extends AbstractDispoImporter { + + private final DispoDataFactory dataFactory; + private final ExecutorAdmin executor; + private final Log logger; + + TmoImporter(DispoDataFactory dataFactory, ExecutorAdmin executor, Log logger) { + this.dataFactory = dataFactory; + this.executor = executor; + this.logger = logger; + } + + @Override + public List importDirectory(Map exisitingItems, File tmoDirectory) { + List toReturn = new LinkedList(); + if (tmoDirectory.isDirectory()) { + TmoFileFilter filter = new TmoFileFilter(); + File[] files = tmoDirectory.listFiles(filter); + List listOfFiles = Arrays.asList(files); + int numThreads = 8; + int partitionSize = listOfFiles.size() / numThreads; + + int remainder = listOfFiles.size() % numThreads; + int startIndex = 0; + int endIndex = 0; + List>> futures = new LinkedList>>(); + for (int i = 0; i < numThreads; i++) { + startIndex = endIndex; + endIndex = startIndex + partitionSize; + if (i == 0) { + endIndex += remainder; + } + List sublist = listOfFiles.subList(startIndex, endIndex); + Worker worker = new Worker(sublist, dataFactory, exisitingItems, this); + Future> future; + try { + future = executor.schedule(worker); + futures.add(future); + } catch (Exception ex) { + logger.error(ex, "Unable to schedule worker"); + } + } + for (Future> future : futures) { + try { + toReturn.addAll(future.get()); + } catch (Exception ex) { + logger.error(ex, "Unable to get future result"); + } + } + } + return toReturn; + } + + private static final class Worker implements Callable> { + + private final List sublist; + private final DispoDataFactory dataFactory; + Map exisitingItems; + private final AbstractDispoImporter importer; + + public Worker(List sublist, DispoDataFactory dataFactory, Map exisitingItems, AbstractDispoImporter importer) { + super(); + this.sublist = sublist; + this.dataFactory = dataFactory; + this.exisitingItems = exisitingItems; + this.importer = importer; + } + + @Override + public List call() throws Exception { + List fromThread = new LinkedList(); + for (File file : sublist) { + InputStream inputStream = null; + try { + inputStream = new FileInputStream(file); + + String scriptName = file.getName().replaceAll("\\..*", ""); + + DispoItemData itemToBuild = new DispoItemData(); + // We already have an item with this name so we now have to check the dates + if (exisitingItems.containsKey(scriptName)) { + DispoItem oldItem = exisitingItems.get(scriptName); + Date lastUpdate = oldItem.getLastUpdate(); + boolean wasSameFile = + DiscrepancyParser.buildItemFromFile(itemToBuild, file.getName(), inputStream, false, lastUpdate); + if (!wasSameFile) { + importer.mergeDispoItems(itemToBuild, oldItem); + fromThread.add(itemToBuild); + } + } else { + DiscrepancyParser.buildItemFromFile(itemToBuild, file.getName(), inputStream, true, new Date()); + dataFactory.initDispoItem(itemToBuild); + fromThread.add(itemToBuild); + } + } finally { + Lib.close(inputStream); + } + } + return fromThread; + } + + }; + + private static final class TmoFileFilter implements FilenameFilter { + + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".tmo"); + } + } + +} diff --git a/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmzImporter.java b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmzImporter.java new file mode 100644 index 00000000000..88d7361654f --- /dev/null +++ b/plugins/org.eclipse.osee.disposition.rest/src/org/eclipse/osee/disposition/rest/internal/importer/TmzImporter.java @@ -0,0 +1,176 @@ +/******************************************************************************* + * Copyright (c) 2014 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.disposition.rest.internal.importer; + +import java.io.File; +import java.io.FileFilter; +import java.io.InputStream; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.eclipse.osee.disposition.model.Discrepancy; +import org.eclipse.osee.disposition.model.DispoItem; +import org.eclipse.osee.disposition.model.DispoItemData; +import org.eclipse.osee.disposition.rest.internal.DispoDataFactory; +import org.eclipse.osee.disposition.rest.util.DispoUtil; +import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; +import org.eclipse.osee.framework.jdk.core.util.GUID; +import org.eclipse.osee.framework.jdk.core.util.Lib; +import org.eclipse.osee.logger.Log; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * @author John Misinco + */ +public class TmzImporter extends AbstractDispoImporter { + + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("MMM dd, yyyy H:mm:ss aa", Locale.US); + + private final Log logger; + private final DispoDataFactory dataFactory; + + public TmzImporter(Log logger, DispoDataFactory dataFactory) { + this.logger = logger; + this.dataFactory = dataFactory; + } + + @Override + public List importDirectory(Map exisitingItems, File filesDir) { + List toReturn = new LinkedList(); + if (!filesDir.exists() || !filesDir.isDirectory()) { + throw new OseeArgumentException("Input directory does not exists or is not a directory [%s]", + filesDir.getAbsolutePath()); + } + + File[] files = filesDir.listFiles(new FileFilter() { + + @Override + public boolean accept(File pathname) { + return Lib.getExtension(pathname.getName()).equals("tmz"); + } + }); + + for (File file : files) { + String scriptName = file.getName().replaceAll("\\..*", ""); + + DispoItem oldItem = exisitingItems.get(scriptName); + Date lastUpdate = oldItem != null ? oldItem.getLastUpdate() : new Date(0); + + DispoItemData itemToBuild = null; + JSONObject discrepancies = null; + ZipFile zf = null; + try { + zf = new ZipFile(file); + ZipEntry entry = zf.getEntry("Overview.json"); + + if (entry != null) { + InputStream inputStream = zf.getInputStream(entry); + String json = Lib.inputStreamToString(inputStream); + itemToBuild = new DispoItemData(); + itemToBuild.setName(scriptName); + processOverview(json, itemToBuild); + + if (oldItem == null || !itemToBuild.getLastUpdate().after(lastUpdate)) { + discrepancies = new JSONObject(); + itemToBuild.setDiscrepanciesList(discrepancies); + entry = zf.getEntry("TestPointSummary.json"); + if (entry != null) { + inputStream = zf.getInputStream(entry); + json = Lib.inputStreamToString(inputStream); + processTestPointSummary(json, discrepancies); + } + + if (oldItem != null) { + mergeDispoItems(itemToBuild, oldItem); + } else { + dataFactory.initDispoItem(itemToBuild); + } + toReturn.add(itemToBuild); + } + } + + } catch (Exception ex) { + logger.info(ex, "Unable to process: [%s]", file.getAbsolutePath()); + } finally { + // ZipFile doesn't implement Closeable in 1.6 + if (zf != null) { + try { + zf.close(); + } catch (Exception ex) { + // do nothing + } + } + } + } + return toReturn; + } + + private void processOverview(String json, DispoItemData dispoItem) throws JSONException, ParseException { + JSONObject record = new JSONObject(json); + JSONObject properties = record.getJSONObject("properties"); + dispoItem.setVersion(properties.getString("version_revision")); + Date date = DATE_FORMAT.parse(properties.getString("version_lastModificationDate")); + dispoItem.setCreationDate(date); + dispoItem.setLastUpdate(date); + } + + private void processTestPointSummary(String json, JSONObject discrepancies) throws JSONException { + JSONObject contents = new JSONObject(json); + JSONArray records = contents.getJSONArray("childRecords"); + for (int i = 0; i < records.length(); i++) { + JSONObject record = records.getJSONObject(i); + int number = record.getInt("number"); + JSONObject testPoint = record.getJSONObject("testPoint"); + boolean passed = testPoint.getBoolean("pass"); + if (!passed) { + Discrepancy discrepancy = new Discrepancy(); + discrepancy.setLocation(number); + String id = GUID.create(); + discrepancy.setId(id); + boolean groupNameIsNull = testPoint.isNull("groupName"); + if (groupNameIsNull) { + String name = testPoint.getString("testPointName"); + String actual = testPoint.getString("actual"); + String expected = testPoint.getString("expected"); + + String text = + String.format("Failure at Test Point %d. Check Point: %s. Expected: %s. Actual: %s. ", number, name, + expected, actual); + discrepancy.setText(text); + } else { + JSONArray testPoints = testPoint.getJSONArray("testPoints"); + StringBuilder text = + new StringBuilder(String.format("Failure at Test Point %d. Check Group with Checkpoint Failures: ", + number)); + for (int j = 0; j < testPoints.length(); j++) { + JSONObject checkPoint = testPoints.getJSONObject(j); + String name = checkPoint.getString("testPointName"); + String actual = checkPoint.getString("actual"); + String expected = checkPoint.getString("expected"); + text.append(String.format("Check Point: %s. Expected: %s. Actual: %s. ", name, expected, actual)); + } + discrepancy.setText(text.toString()); + } + JSONObject discrepancyAsJson = DispoUtil.discrepancyToJsonObj(discrepancy); + discrepancies.put(id, discrepancyAsJson); + } + } + } + +} -- cgit v1.2.3