Skip to main content
summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRyan D. Brooks2019-12-21 17:32:17 +0000
committerRyan D. Brooks2020-12-15 21:40:48 +0000
commit00dc5edb3e0c89ed11c958ac146bf10bd2431487 (patch)
tree71ac4dc8310cfb85a01913da7fbc65188f703bc7
parente7ce70251002041b21e767020eca3bd119bbbabe (diff)
downloadorg.eclipse.osee-00dc5edb3e0c89ed11c958ac146bf10bd2431487.tar.gz
org.eclipse.osee-00dc5edb3e0c89ed11c958ac146bf10bd2431487.tar.xz
org.eclipse.osee-00dc5edb3e0c89ed11c958ac146bf10bd2431487.zip
feature: Add purge unused binary data admin operation
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/DataStoreAdmin.java2
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/OrcsAdminImpl.java5
-rw-r--r--plugins/org.eclipse.osee.orcs.db/META-INF/MANIFEST.MF1
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/DataStoreAdminImpl.java154
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/OrcsDataStoreImpl.java2
-rw-r--r--plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/DatastoreEndpoint.java5
-rw-r--r--plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/DatastoreEndpointImpl.java5
-rw-r--r--plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/OrcsAdmin.java2
8 files changed, 174 insertions, 2 deletions
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/DataStoreAdmin.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/DataStoreAdmin.java
index e0c9f131fee..bcc52e3615a 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/DataStoreAdmin.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/DataStoreAdmin.java
@@ -37,4 +37,6 @@ public interface DataStoreAdmin {
void updateBootstrapUser(UserId accountId);
+ long purgeUnusedBinaryAttributes();
+
}
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/OrcsAdminImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/OrcsAdminImpl.java
index ce7905abc2b..b6ce412925e 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/OrcsAdminImpl.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/OrcsAdminImpl.java
@@ -262,4 +262,9 @@ public class OrcsAdminImpl implements OrcsAdmin {
public void updateBootstrapUser(UserId accountId) {
dataStoreAdmin.updateBootstrapUser(accountId);
}
+
+ @Override
+ public long purgeUnusedBinaryAttributes() {
+ return dataStoreAdmin.purgeUnusedBinaryAttributes();
+ }
} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.db/META-INF/MANIFEST.MF b/plugins/org.eclipse.osee.orcs.db/META-INF/MANIFEST.MF
index fe73b8f5ba4..18e020373cb 100644
--- a/plugins/org.eclipse.osee.orcs.db/META-INF/MANIFEST.MF
+++ b/plugins/org.eclipse.osee.orcs.db/META-INF/MANIFEST.MF
@@ -28,6 +28,7 @@ Import-Package: com.google.common.base,
org.eclipse.osee.framework.core.model.type,
org.eclipse.osee.framework.core.operation,
org.eclipse.osee.framework.core.sql,
+ org.eclipse.osee.framework.core.util,
org.eclipse.osee.framework.jdk.core.result,
org.eclipse.osee.framework.jdk.core.text,
org.eclipse.osee.framework.jdk.core.text.change,
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/DataStoreAdminImpl.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/DataStoreAdminImpl.java
index cc20e70fd28..8f4dd21aaad 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/DataStoreAdminImpl.java
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/DataStoreAdminImpl.java
@@ -14,11 +14,22 @@
package org.eclipse.osee.orcs.db.internal;
import com.google.common.base.Supplier;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
import org.eclipse.osee.framework.core.data.UserId;
+import java.util.stream.Stream;
import org.eclipse.osee.framework.core.enums.PermissionEnum;
import org.eclipse.osee.framework.jdk.core.util.Conditions;
import org.eclipse.osee.framework.jdk.core.util.GUID;
@@ -34,7 +45,10 @@ import org.eclipse.osee.orcs.core.ds.DataStoreConstants;
import org.eclipse.osee.orcs.core.ds.DataStoreInfo;
import org.eclipse.osee.orcs.db.internal.callable.FetchDatastoreInfoCallable;
import org.eclipse.osee.orcs.db.internal.callable.MigrateDatastoreCallable;
+import org.eclipse.osee.orcs.db.internal.resource.AttributeLocatorProvider;
import org.eclipse.osee.orcs.db.internal.resource.ResourceConstants;
+import org.eclipse.osee.orcs.db.internal.sql.join.IdJoinQuery;
+import org.eclipse.osee.orcs.db.internal.sql.join.SqlJoinFactory;
import org.eclipse.osee.orcs.db.internal.util.DynamicSchemaResourceProvider;
/**
@@ -45,11 +59,13 @@ public class DataStoreAdminImpl implements DataStoreAdmin {
private final Log logger;
private final JdbcClient jdbcClient;
private final SystemProperties properties;
+ private final SqlJoinFactory sqlJoinFactory;
- public DataStoreAdminImpl(Log logger, JdbcClient jdbcClient, SystemProperties properties) {
+ public DataStoreAdminImpl(Log logger, JdbcClient jdbcClient, SystemProperties properties, SqlJoinFactory sqlJoinFactory) {
this.logger = logger;
this.jdbcClient = jdbcClient;
this.properties = properties;
+ this.sqlJoinFactory = sqlJoinFactory;
}
@Override
@@ -104,4 +120,140 @@ public class DataStoreAdminImpl implements DataStoreAdmin {
public void updateBootstrapUser(UserId accountId) {
jdbcClient.runPreparedUpdate("UPDATE osee_tx_details SET author = ? where author <= 0", accountId);
}
+ private void recurseDir(File parentDirectory) throws IOException {
+ File[] files = parentDirectory.listFiles();
+ boolean hasSubDirectory = false;
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].isDirectory()) {
+ hasSubDirectory = true;
+ recurseDir(files[i]);
+ }
+ }
+ if (!hasSubDirectory) {
+
+ }
+ }
+
+ private Path attributeDataPath;
+ private IdJoinQuery gammaJoin;
+ private long countPurged = 0;
+
+ private void walker() throws IOException {
+ Path attributeDataPath = Paths.get(ResourceConstants.getAttributeDataPath(preferences));
+
+ Files.walkFileTree(attributeDataPath, new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
+ if (e == null) {
+ Files.delete(dir);
+ return FileVisitResult.CONTINUE;
+ } else {
+ // directory iteration failed
+ throw e;
+ }
+ }
+ });
+
+ }
+
+ private void handleFile(Path path) {
+ File file = path.toFile();
+ if (!file.isDirectory()) {
+ Path pathRelative = attributeDataPath.relativize(path);
+ StringBuilder strB = new StringBuilder();
+ int stop = pathRelative.getNameCount() - 1;
+ for (int i = 0; i < stop; i++) {
+ strB.append(pathRelative.getName(i));
+ }
+ gammaJoin.add(Long.valueOf(strB.toString()));
+ }
+ }
+
+ @Override
+ public long purgeUnusedBinaryAttributes() {
+ // return loadBinaryGammasIntoJoinTable();
+ findUnusedGammas();
+ return countPurged;
+
+ }
+
+ private long loadBinaryGammasIntoJoinTable() {
+ try (IdJoinQuery idJoin = sqlJoinFactory.createIdJoinQuery()) {
+ gammaJoin = idJoin;
+ try (Stream<String> stream = Files.lines(Paths.get("C:\\Users\\b1117361\\Desktop\\g4.txt"))) {
+ stream.forEach(gamma -> gammaJoin.add(Long.valueOf(gamma)));
+ } catch (IOException ex) {
+ logger.error(ex, "in purgeUnusedBinaryAttributes");
+ }
+ gammaJoin.store();
+ System.out.println(gammaJoin.getQueryId());
+ System.out.println(gammaJoin.getQueryId());
+ // don't let try statement complete or the join table will be cleaned out
+ }
+ return gammaJoin.size();
+ }
+
+ private void findAllBinaryGammas() {
+ try (IdJoinQuery idJoin = sqlJoinFactory.createIdJoinQuery()) {
+ gammaJoin = idJoin;
+ // String attributeDataPathStr = ResourceConstants.getAttributeDataPath(preferences);
+ String attributeDataPathStr = "\\\\osee.msc.az.boeing.com\\osee_server_data\\attr\\975";
+ attributeDataPath = Paths.get(attributeDataPathStr);
+
+ try (Stream<Path> walk = Files.walk(attributeDataPath)) {
+ walk.sorted(Comparator.reverseOrder()).forEach(this::handleFile);
+ }
+ gammaJoin.store();
+
+ } catch (IOException ex) {
+ logger.error(ex, "in purgeUnusedBinaryAttributes");
+ }
+ }
+
+ private void findUnusedGammas() {
+ String attributeDataPathStr = ResourceConstants.getAttributeDataPath(preferences);
+ String sql = "SELECT id FROM osee_join_id WHERE query_id = ? MINUS SELECT gamma_id FROM osee_txs";
+
+ try (BufferedWriter out = new BufferedWriter(new FileWriter("C:\\Users\\b1117361\\Desktop\\remove11.sh"))) {
+
+ jdbcClient.runQuery(stmt -> {
+ String gammaId = stmt.getString("id");
+ StringBuilder builder = new StringBuilder();
+ AttributeLocatorProvider.seedTo(builder, gammaId);
+
+ // File file = new File(attributeDataFile, builder.toString());
+ // Path removePath = Paths.get(attributeDataPathStr, builder.toString());
+ try {
+ out.write("rm -rf " + builder + "\n");
+ } catch (IOException ex) {
+ logger.error(ex, "in findUnusedGammas");
+ }
+ countPurged++;
+ // Lib.deleteFileAndEmptyParents(attributeDataPathStr, file);
+ }, sql, 726199745L /* gammaJoin.getQueryId() */);
+ out.flush();
+ } catch (IOException ex) {
+ logger.error(ex, "in findUnusedGammas");
+ }
+ }
+
+ private void deleteDirectory(Path removePath) {
+ try {
+ Files.walk(removePath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+ } catch (IOException ex) {
+ logger.error(ex, "in purgeUnusedBinaryAttributes");
+ }
+ }
+
} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/OrcsDataStoreImpl.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/OrcsDataStoreImpl.java
index 913f5e533f7..893000f87b4 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/OrcsDataStoreImpl.java
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/OrcsDataStoreImpl.java
@@ -94,7 +94,7 @@ public class OrcsDataStoreImpl implements OrcsDataStore {
TxModule txModule = new TxModule(logger, jdbcClient, joinFactory, idManager);
- DataStoreAdmin adminModule = new DataStoreAdminImpl(logger, jdbcClient, properties);
+ DataStoreAdmin adminModule = new DataStoreAdminImpl(logger, jdbcClient, properties, joinFactory);
KeyValueModule keyValueModule = new KeyValueModule(jdbcClient);
diff --git a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/DatastoreEndpoint.java b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/DatastoreEndpoint.java
index 5053d3995e5..1797909f288 100644
--- a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/DatastoreEndpoint.java
+++ b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/DatastoreEndpoint.java
@@ -14,6 +14,7 @@
package org.eclipse.osee.orcs.rest.model;
import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
@@ -65,4 +66,8 @@ public interface DatastoreEndpoint {
@Produces(MediaType.APPLICATION_JSON)
void updateBootstrapUser();
+ @DELETE
+ @Path("unused/blobs")
+ @Produces(MediaType.APPLICATION_JSON)
+ long purgeUnusedBinaryAttributes();
} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/DatastoreEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/DatastoreEndpointImpl.java
index 62cfb26f56f..18052c1e42c 100644
--- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/DatastoreEndpointImpl.java
+++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/DatastoreEndpointImpl.java
@@ -113,4 +113,9 @@ public class DatastoreEndpointImpl implements DatastoreEndpoint {
public void updateBootstrapUser() {
adminOps.updateBootstrapUser(accountId);
}
+
+ @Override
+ public long purgeUnusedBinaryAttributes() {
+ return adminOps.purgeUnusedBinaryAttributes();
+ }
} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/OrcsAdmin.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/OrcsAdmin.java
index f6fddb81168..e34d1b88ffb 100644
--- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/OrcsAdmin.java
+++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/OrcsAdmin.java
@@ -51,4 +51,6 @@ public interface OrcsAdmin {
void registerMissingOrcsTypeJoins();
+ long purgeUnusedBinaryAttributes();
+
} \ No newline at end of file

Back to the top