Skip to main content
summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRyan D. Brooks2018-02-27 14:15:51 +0000
committerRyan D. Brooks2020-12-15 22:27:47 +0000
commite401af2380dbd5bfae80a53fa789ada18d7a352d (patch)
treeb964e6c9eda1c3587408f81ac10f911a45c07b61
parent9f81f171bd2e8c493d46e063d980a0f1905b6558 (diff)
downloadorg.eclipse.osee-e401af2380dbd5bfae80a53fa789ada18d7a352d.tar.gz
org.eclipse.osee-e401af2380dbd5bfae80a53fa789ada18d7a352d.tar.xz
org.eclipse.osee-e401af2380dbd5bfae80a53fa789ada18d7a352d.zip
feature: Create faster query indexer
- Provides a faster and more flexible approach for indexing the - attribute values for various kinds of search including whole match, - tokenized by whitespace, and tokenized by non-alphanumeric. Any one - of the proceeding options can be combined with case sensitivity. Change-Id: I97077898e9827786c806460afbb98301c1d5ebbd
-rw-r--r--plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java8
-rw-r--r--plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java8
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java20
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java5
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java67
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java12
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java14
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java216
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java153
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java2
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java13
-rw-r--r--plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java11
-rw-r--r--plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java1
-rw-r--r--plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java13
-rw-r--r--plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java11
-rw-r--r--plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java9
16 files changed, 546 insertions, 17 deletions
diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java
index 84ef46dde10..8456cd6f596 100644
--- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java
+++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java
@@ -17,6 +17,7 @@ package org.eclipse.osee.framework.core.enums;
* @author John Misinco
*/
public enum QueryOption {
+ NONE,
CASE__MATCH,
CASE__IGNORE,
@@ -30,7 +31,12 @@ public enum QueryOption {
// matching the token order
TOKEN_MATCH_ORDER__ANY,
- TOKEN_MATCH_ORDER__MATCH;
+ TOKEN_MATCH_ORDER__MATCH,
+
+ // can be used to match one value or any values (i.e. using OR) but not all values (i.e. using AND)
+ WHOLE_MATCH,
+ TOKENIZE_WHITESPACE,
+ TOKENIZE_NON_ALPHANUMERIC;
public static QueryOption getTokenOrderType(boolean matchOrder) {
return matchOrder ? TOKEN_MATCH_ORDER__MATCH : TOKEN_MATCH_ORDER__ANY;
diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java
index fb08444ed1c..1eaa6c21e74 100644
--- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java
+++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java
@@ -680,6 +680,14 @@ public class SqlTable extends NamedBase {
OSEE_OAUTH_TOKEN_TABLE.createIndex("OSEE_OAUTH_TOKEN__TK_IDX", false, OSEE_OAUTH_TOKEN_TOKEN_KEY.getName());
}
+ public static final SqlTable SEARCH_HASH = new SqlTable("osee_search_hash", "sh", 1);
+ public static final SqlColumn SEARCH_HASH_APP_ID = TXS_TABLE.addColumn("APP_ID", JDBCType.BIGINT);
+ public static final SqlColumn SEARCH_HASH_HASH = TXS_TABLE.addColumn("HASH", JDBCType.BIGINT);
+ public static final SqlColumn SEARCH_HASH_GAMMA_ID = TXS_TABLE.addColumn("GAMMA_ID", JDBCType.BIGINT);
+ static {
+ TXS_TABLE.setPrimaryKeyConstraint(SEARCH_HASH_APP_ID, SEARCH_HASH_HASH, SEARCH_HASH_GAMMA_ID);
+ }
+
private final String aliasPrefix;
private final ObjectType objectType;
private final ChainingArrayList<@NonNull SqlColumn> columns;
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java
index ddc35d233cc..53a84ac99c9 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java
@@ -54,6 +54,7 @@ import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactGuids;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactTxComment;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType;
+import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists;
@@ -287,8 +288,10 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch {
return branch;
}
- public void select(AttributeTypeToken attributeType) {
+ @Override
+ public QueryBuilder select(AttributeTypeToken attributeType) {
this.attributeType = attributeType;
+ return this;
}
@Override
@@ -501,6 +504,21 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch {
}
@Override
+ public QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) {
+ return addAndCheck(new CriteriaAttributeHash(attributeType, value, option, caseSensitive));
+ }
+
+ @Override
+ public QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) {
+ return addAndCheck(new CriteriaAttributeHash(attributeType, values, option, caseSensitive));
+ }
+
+ @Override
+ public QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive) {
+ return addAndCheck(new CriteriaAttributeHash(value, option, caseSensitive));
+ }
+
+ @Override
public QueryBuilder andAttributeIs(AttributeTypeId attributeType, String... values) {
return andAttributeIs(attributeType, Arrays.asList(values));
}
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java
index 1d36b335ab1..6d77fcdbe45 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java
@@ -18,6 +18,7 @@ import java.util.Set;
import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.OrcsTokenService;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
import org.eclipse.osee.orcs.OrcsSession;
import org.eclipse.osee.orcs.search.IndexerCollector;
@@ -46,4 +47,8 @@ public interface QueryEngineIndexer {
void indexAttrTypeIds(OrcsSession session, OrcsTokenService tokenService, Iterable<Long> attrTypeIds);
void indexAttrTypeMissingOnly(OrcsTokenService tokenService, Iterable<Long> attrTypeIds);
+
+ int createTermHashes(Iterable<GammaId> gammaIds);
+
+ int createTermHashes(AttributeTypeId attributeType);
} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java
new file mode 100644
index 00000000000..b960a673ff3
--- /dev/null
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java
@@ -0,0 +1,67 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Boeing.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Boeing - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.osee.orcs.core.ds.criteria;
+
+import java.util.Collection;
+import java.util.Collections;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
+import org.eclipse.osee.framework.core.enums.QueryOption;
+import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException;
+import org.eclipse.osee.orcs.core.ds.Criteria;
+import org.eclipse.osee.orcs.core.ds.Options;
+
+/**
+ * @author Ryan D. Brooks
+ */
+public class CriteriaAttributeHash extends Criteria {
+ private final AttributeTypeId attributeType;
+ private final Collection<String> values;
+ private final QueryOption option;
+ private final boolean caseSensitive;
+
+ public CriteriaAttributeHash(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) {
+ this.attributeType = attributeType;
+ this.values = values;
+ this.option = option;
+ this.caseSensitive = caseSensitive;
+ }
+
+ public CriteriaAttributeHash(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) {
+ this(attributeType, Collections.singletonList(value), option, caseSensitive);
+ }
+
+ public CriteriaAttributeHash(String value, QueryOption option, boolean caseSensitive) {
+ this(AttributeTypeId.SENTINEL, value, option, caseSensitive);
+ }
+
+ public AttributeTypeId getAttributeType() {
+ return attributeType;
+ }
+
+ public Collection<String> getValues() {
+ return values;
+ }
+
+ public QueryOption getOption() {
+ return option;
+ }
+
+ public boolean isCaseSensitive() {
+ return caseSensitive;
+ }
+
+ @Override
+ public void checkValid(Options options) {
+ if (values.isEmpty()) {
+ throw new OseeArgumentException("CriteriaAttributeHash requires at least one value to match");
+ }
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java
index f7750b3fe5c..2c7f02cbbf6 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java
@@ -20,6 +20,7 @@ import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.OrcsTokenService;
import org.eclipse.osee.framework.core.data.AttributeTypeGeneric;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
import org.eclipse.osee.orcs.OrcsSession;
import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer;
@@ -90,4 +91,13 @@ public class QueryIndexerImpl implements QueryIndexer {
return engineIndexer.purgeAllIndexes(session);
}
-}
+ @Override
+ public int createTermHashes(Iterable<GammaId> gammaIds) {
+ return engineIndexer.createTermHashes(gammaIds);
+ }
+
+ @Override
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return engineIndexer.createTermHashes(attributeType);
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java
index 0a4fbbcfaa7..f5374a87a42 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java
@@ -42,12 +42,12 @@ import org.eclipse.osee.framework.core.enums.CoreArtifactTokens;
import org.eclipse.osee.framework.core.enums.CoreArtifactTypes;
import org.eclipse.osee.framework.core.enums.CoreAttributeTypes;
import org.eclipse.osee.framework.core.enums.CoreTupleTypes;
+import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.core.enums.RelationSorter;
import org.eclipse.osee.framework.core.enums.SqlTable;
import org.eclipse.osee.framework.jdk.core.type.Id;
import org.eclipse.osee.framework.jdk.core.type.NamedId;
import org.eclipse.osee.framework.jdk.core.type.OseeCoreException;
-import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.framework.jdk.core.util.Conditions;
import org.eclipse.osee.orcs.KeyValueOps;
import org.eclipse.osee.orcs.OrcsApi;
@@ -172,12 +172,12 @@ public class TransactionBuilderImpl implements TransactionBuilder {
@Override
public List<ArtifactToken> createArtifacts(ArtifactTypeToken artifactType, ArtifactId parent, List<String> names) {
- ResultSet<ArtifactReadable> results =
- queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).and(CoreAttributeTypes.Name,
- names).getResults();
- if (!results.isEmpty()) {
- throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", results.size(),
- artifactType, results.getList());
+ List<ArtifactToken> duplicates =
+ queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).andAttribute(CoreAttributeTypes.Name, names,
+ QueryOption.TOKENIZE_WHITESPACE, false).loadArtifactTokens();
+ if (!duplicates.isEmpty()) {
+ throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", duplicates.size(),
+ artifactType, duplicates);
}
List<ArtifactToken> tokens = new ArrayList<>(names.size());
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java
new file mode 100644
index 00000000000..337daab8f0a
--- /dev/null
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java
@@ -0,0 +1,216 @@
+/*******************************************************************************
+ * Copyright (c) 218 Boeing.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Boeing - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.osee.orcs.db.internal.search;
+
+import static org.eclipse.osee.jdbc.JdbcConstants.JDBC__MAX_FETCH_SIZE;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
+import org.eclipse.osee.framework.core.data.GammaId;
+import org.eclipse.osee.framework.core.enums.QueryOption;
+import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException;
+import org.eclipse.osee.jdbc.JdbcClient;
+import org.eclipse.osee.jdbc.JdbcStatement;
+import org.eclipse.osee.jdbc.OseePreparedStatement;
+import org.eclipse.osee.orcs.db.internal.sql.join.IdJoinQuery;
+import org.eclipse.osee.orcs.db.internal.sql.join.SqlJoinFactory;
+
+/**
+ * Instances of this class should not be shared among threads. Use multiple term normalization rules and then store the
+ * unique set of resulting hashes. Each normalization starts with making all letters lower-case.</br>
+ * <li>whole match = no tokenize; keep only alpha-numeric characters
+ * <li>original = tokenize on characters that are not digits or letters
+ * <li>partial with fuzzy punc = tokenize on whitespace; strip all punctuation
+ * <li>maybe not = tokenize on whitespace; strip leading and trailing punctuation; replace remaining punctuation with
+ * spaces
+ * <ul>
+ * <li>{Requirement_Name}
+ * <li>use new tablespace called osee_search Replace TagProcessor and use XmlTextInputStream
+ *
+ * @author Ryan D. Brooks
+ */
+public class SearchTermHash {
+ private static final String SELECT_MISSING_ATTRIBUTES =
+ "SELECT gamma_id, value FROM osee_attribute att WHERE attr_type_id = ? AND value is not null AND not exists (SELECT 1 FROM osee_search_hash has WHERE has.gamma_id = att.gamma_id)";
+
+ private static final String SELECT_ATTRIBUTES_BY_TYPE =
+ "SELECT gamma_id, value FROM osee_attribute WHERE attr_type_id = ? AND value is not null";
+
+ private static final String SELECT_ATTRIBUTES_BY_GAMMA =
+ "SELECT att.gamma_id, value FROM osee_attribute att, osee_join_id WHERE query_id = ? AND id = gamma_id AND value is not null";
+
+ private final SqlJoinFactory joinFactory;
+ private final JdbcClient jdbcClient;
+ private OseePreparedStatement insertStatement;
+ private final HashSet<Long> hashes = new HashSet<>();
+ private final List<List<Long>> resultHashes = new ArrayList<>();
+ private final List<Long> singleTokens = new ArrayList<>();
+
+ public SearchTermHash() {
+ this(null, null);
+ }
+
+ public SearchTermHash(JdbcClient jdbcClient, SqlJoinFactory joinFactory) {
+ this.jdbcClient = jdbcClient;
+ this.joinFactory = joinFactory;
+ }
+
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return createTermHashes(SELECT_ATTRIBUTES_BY_TYPE, attributeType);
+ }
+
+ public int createTermHashes(Iterable<GammaId> gammaIds) {
+ try (IdJoinQuery joinQuery = joinFactory.createIdJoinQuery()) {
+ joinQuery.addAndStore(gammaIds);
+ return createTermHashes(SELECT_ATTRIBUTES_BY_GAMMA, joinQuery.getQueryId());
+ }
+ }
+
+ private int createTermHashes(String sql, Object... data) {
+ insertStatement =
+ jdbcClient.getBatchStatement("INSERT INTO osee_search_hash (app_id, hash, gamma_id) VALUES (?, ?, ?)");
+ int loaded = jdbcClient.runQuery(this::createTermHashes, JDBC__MAX_FETCH_SIZE, sql, data);
+ insertStatement.execute();
+ return loaded;
+ }
+
+ public List<List<Long>> getTermHashes(Collection<String> values, QueryOption queryOption) {
+ resultHashes.clear();
+ singleTokens.clear();
+ resultHashes.add(singleTokens);
+ for (String value : values) {
+ switch (queryOption) {
+ case TOKENIZE_NON_ALPHANUMERIC:
+ addOriginalMatchHash(value);
+ break;
+ case TOKENIZE_WHITESPACE:
+ addPartialFuzzyPuncHash(value);
+ break;
+ case WHOLE_MATCH:
+ getWholeMatchHash(value);
+ break;
+ default:
+ throw new OseeArgumentException("Unexpected query option %s", queryOption);
+ }
+ copyHashes();
+ }
+ return resultHashes;
+ }
+
+ private void copyHashes() {
+ if (hashes.size() == 1) {
+ singleTokens.add(hashes.iterator().next());
+ } else {
+ resultHashes.add(new ArrayList<>(hashes));
+ }
+ hashes.clear();
+ }
+
+ private void createTermHashes(JdbcStatement stmt) {
+ String value = stmt.getString("value");
+ addOriginalMatchHash(value);
+ addPartialFuzzyPuncHash(value);
+ getWholeMatchHash(value);
+
+ for (Long hash : hashes) {
+ insertStatement.addToBatch(1, hash, stmt.getLong("gamma_id"));
+ }
+ hashes.clear();
+ }
+
+ /**
+ * don't tokenize and include only alpha-numeric characters in hash. If the value contains only non alpha-numeric
+ * characters, then the hash is computed using all of its characters.
+ */
+ public void getWholeMatchHash(String value) {
+ long hash = 0;
+ for (char c : value.toCharArray()) {
+ c = toLower(c);
+ if (isLetterOrDigit(c)) {
+ hash = 31 * hash + c;
+ }
+ }
+
+ hashes.add(handleNoAlphaNumeric(value, hash));
+ }
+
+ private static long handleNoAlphaNumeric(String value, long hash) {
+ if (hash == 0) {
+ for (char c : value.toCharArray()) {
+ hash = 31 * hash + toLower(c);
+ }
+ }
+ return hash;
+ }
+
+ /**
+ * tokenize on everything except alpha-numeric characters
+ */
+ private void addOriginalMatchHash(String value) {
+ long hash = 0;
+ for (char c : value.toCharArray()) {
+ c = toLower(c);
+ if (isLetterOrDigit(c)) {
+ hash = 31 * hash + c;
+ } else {
+ addHash(hash);
+ hash = 0;
+ }
+ }
+ addHash(hash);
+ }
+
+ /**
+ * tokenize on whitespace and include only alpha-numeric characters in hash
+ */
+ private void addPartialFuzzyPuncHash(String value) {
+ long hash = 0;
+ for (char c : value.toCharArray()) {
+ if (c == ' ' || c == '\n' || c == '\r' || c == '\t') {
+ addHash(hash);
+ hash = 0;
+ } else {
+ c = toLower(c);
+ if (isLetterOrDigit(c)) {
+ hash = 31 * hash + c;
+ }
+ }
+ }
+ addHash(hash);
+ }
+
+ private void addHash(Long hash) {
+ if (hash != 0) {
+ hashes.add(hash);
+ }
+ }
+
+ private static boolean isLetterOrDigit(char c) {
+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z');
+ }
+
+ private static char toLower(char c) {
+ if (c >= 'A' && c <= 'Z') {
+ return (char) (c + 32);
+ }
+ return c;
+ }
+
+ public static void main(String[] args) {
+ SearchTermHash app = new SearchTermHash();
+ System.out.println(app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.WHOLE_MATCH));
+ System.out.println(
+ app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.TOKENIZE_WHITESPACE));
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java
new file mode 100644
index 00000000000..1c74ec832fe
--- /dev/null
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java
@@ -0,0 +1,153 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Boeing.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Boeing - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.osee.orcs.db.internal.search.handlers;
+
+import java.util.Iterator;
+import java.util.List;
+import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash;
+import org.eclipse.osee.orcs.db.internal.search.SearchTermHash;
+import org.eclipse.osee.orcs.db.internal.sql.AbstractSqlWriter;
+import org.eclipse.osee.orcs.db.internal.sql.SqlHandler;
+import org.eclipse.osee.orcs.db.internal.sql.TableEnum;
+
+/**
+ * @author Ryan D. Brooks
+ */
+public class AttributeHashQueryHandler extends SqlHandler<CriteriaAttributeHash> {
+ private String attTxsAlias;
+ private String attAlias;
+ private String artAlias;
+ private CriteriaAttributeHash criteria;
+ private final SearchTermHash hasher = new SearchTermHash();
+ private List<List<Long>> hashes;
+
+ @Override
+ public void setData(CriteriaAttributeHash criteria) {
+ this.criteria = criteria;
+ }
+
+ @Override
+ public void addTables(AbstractSqlWriter writer) {
+ hashes = hasher.getTermHashes(criteria.getValues(), criteria.getOption());
+ int hashTableCount = countNeededHashTables();
+
+ for (int i = 0; i < hashTableCount; i++) {
+ writer.addTable(TableEnum.SEARCH_HASH_TABLE);
+ }
+ attAlias = writer.addTable(TableEnum.ATTRIBUTE_TABLE);
+ artAlias = writer.getMainTableAlias(TableEnum.ARTIFACT_TABLE);
+ attTxsAlias = writer.addTable(TableEnum.TXS_TABLE);
+ }
+
+ private int countNeededHashTables() {
+ int hashTableCount = 0;
+ Iterator<List<Long>> iterator = hashes.iterator();
+ List<Long> singleTokens = iterator.next();
+ if (!singleTokens.isEmpty()) {
+ hashTableCount++;
+ }
+ while (iterator.hasNext()) {
+ List<Long> list = iterator.next();
+ hashTableCount += list.size();
+ }
+ return hashTableCount;
+ }
+
+ /**
+ * Combine all single tokens together using a simple in clause
+ */
+ private void writeHashTableForSingleTokens(AbstractSqlWriter writer, String hshAlias, List<Long> singleTokens) {
+ writer.write("(");
+ writer.writeEqualsParameterAnd(hshAlias, "app_id", 1);
+ if (singleTokens.size() == 1) {
+ writer.writeEqualsParameter(hshAlias, "hash", singleTokens.get(0));
+ } else {
+ writer.write(hshAlias);
+ writer.write(".hash IN (");
+ boolean notFirst = false;
+ for (Long hash : singleTokens) {
+ if (notFirst) {
+ writer.write(", ?");
+ } else {
+ writer.write("?");
+ notFirst = true;
+ }
+ writer.addParameter(hash);
+ }
+ writer.write(")");
+ }
+ writer.write(" AND ");
+ writer.writeEquals(hshAlias, attAlias, "gamma_id");
+ writer.write(")");
+ }
+
+ private void addHashPredicates(AbstractSqlWriter writer) {
+ Iterator<String> aliases = writer.getAliases(TableEnum.SEARCH_HASH_TABLE).iterator();
+ Iterator<List<Long>> hashIterator = hashes.iterator();
+ String prevHshAlias = aliases.next();
+ List<Long> singleTokens = hashIterator.next();
+
+ writer.write("(");
+ boolean hasSingleTokens = !singleTokens.isEmpty();
+ if (hasSingleTokens) {
+ writeHashTableForSingleTokens(writer, prevHshAlias, singleTokens);
+ }
+
+ boolean useOr = hasSingleTokens;
+ while (hashIterator.hasNext()) {
+ if (useOr) {
+ writer.write(" OR\n");
+ } else {
+ useOr = true;
+ }
+ writer.write(" (");
+
+ boolean joinGammaIds = false;
+ for (Long hash : hashIterator.next()) {
+ String hshAlias = aliases.next();
+ if (joinGammaIds) {
+ writer.writeEquals(prevHshAlias, hshAlias, "gamma_id");
+ writer.write(" AND ");
+ } else {
+ joinGammaIds = true;
+ }
+ writer.write(hshAlias);
+ writer.write(".app_id = ? AND ");
+ writer.addParameter(1);
+ writer.write(hshAlias);
+ writer.write(".hash = ? AND ");
+ writer.addParameter(hash);
+ prevHshAlias = hshAlias;
+ }
+ writer.writeEquals(prevHshAlias, attAlias, "gamma_id");
+ writer.write(")");
+ }
+ writer.write(") AND\n ");
+ }
+
+ @Override
+ public void addPredicates(AbstractSqlWriter writer) {
+ addHashPredicates(writer);
+ if (criteria.getAttributeType().isValid()) {
+ writer.writeEqualsParameterAnd(attAlias, "attr_type_id", criteria.getAttributeType());
+ }
+ writer.writeEquals(attAlias, artAlias, "art_id");
+ writer.write(" AND ");
+ writer.writeEquals(attAlias, attTxsAlias, "gamma_id");
+ writer.write(" AND ");
+ writer.write(writer.getTxBranchFilter(attTxsAlias));
+ }
+
+ @Override
+ public int getPriority() {
+ return SqlHandlerPriority.ATTRIBUTE_TOKENIZED_VALUE.ordinal();
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java
index 3304a1b6bcb..917438e58f8 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java
@@ -24,6 +24,7 @@ import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactTxComment;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAssociatedArtId;
+import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists;
@@ -96,6 +97,7 @@ public final class SqlHandlerFactoryUtil {
}
private static void addArtifactHandlers(Map<Class<? extends Criteria>, Class<? extends SqlHandler<?>>> handleMap) {
+ handleMap.put(CriteriaAttributeHash.class, AttributeHashQueryHandler.class);
handleMap.put(CriteriaArtifactGuids.class, ArtifactGuidSqlHandler.class);
handleMap.put(CriteriaArtifactIds.class, ArtifactIdsSqlHandler.class);
handleMap.put(CriteriaArtifactType.class, ArtifactTypeSqlHandler.class);
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java
index cd9e510a4f2..e5ad4d3d8c4 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java
@@ -19,6 +19,7 @@ import java.util.Set;
import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.OrcsTokenService;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
import org.eclipse.osee.framework.jdk.core.type.OseeCoreException;
import org.eclipse.osee.framework.jdk.core.util.Collections;
@@ -29,6 +30,7 @@ import org.eclipse.osee.logger.Log;
import org.eclipse.osee.orcs.OrcsSession;
import org.eclipse.osee.orcs.core.ds.IndexerData;
import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer;
+import org.eclipse.osee.orcs.db.internal.search.SearchTermHash;
import org.eclipse.osee.orcs.db.internal.search.indexer.callable.DeleteTagSetDatabaseTxCallable;
import org.eclipse.osee.orcs.db.internal.search.indexer.callable.IndexerDatabaseStatisticsCallable;
import org.eclipse.osee.orcs.db.internal.search.indexer.callable.PurgeAllTagsDatabaseCallable;
@@ -157,4 +159,13 @@ public class QueryEngineIndexerImpl implements QueryEngineIndexer {
systemCollector.removeCollector(collector);
}
-}
+ @Override
+ public int createTermHashes(Iterable<GammaId> gammaIds) {
+ return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(gammaIds);
+ }
+
+ @Override
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(attributeType);
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java
index a797002d14f..a5ec89fe6ff 100644
--- a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java
+++ b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java
@@ -13,15 +13,19 @@
package org.eclipse.osee.orcs.rest.model;
+import java.util.List;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
+import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
+import org.eclipse.osee.framework.core.data.GammaId;
/**
* @author Roberto E. Escobar
@@ -39,6 +43,13 @@ public interface IndexerEndpoint {
@Consumes(MediaType.APPLICATION_JSON)
Response indexResources(IndexResources options);
+ @POST
+ @Path("attributeType/{attributeType}")
+ int createTermHashes(@PathParam("attributeType") AttributeTypeId attributeType);
+
+ @POST
+ int createTermHashes(List<GammaId> gammaIds);
+
@DELETE
@Path("queue")
Response deleteIndexQueue();
diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java
index e941503b071..ac17a31ff1c 100644
--- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java
+++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java
@@ -30,6 +30,7 @@ import org.eclipse.osee.framework.core.data.BranchId;
import org.eclipse.osee.framework.core.data.TransactionId;
import org.eclipse.osee.framework.core.data.TransactionToken;
import org.eclipse.osee.framework.core.data.UserId;
+import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.jdk.core.type.MatchLocation;
import org.eclipse.osee.framework.jdk.core.type.MultipleItemsExist;
import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException;
diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java
index bf81aacbae9..671134dcfa0 100644
--- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java
+++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java
@@ -22,8 +22,10 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
import org.eclipse.osee.framework.core.data.BranchId;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.framework.jdk.core.util.Collections;
import org.eclipse.osee.jaxrs.OseeWebApplicationException;
@@ -104,4 +106,13 @@ public class IndexerEndpointImpl implements IndexerEndpoint {
return asResponse(modified);
}
-}
+ @Override
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return getIndexer().createTermHashes(attributeType);
+ }
+
+ @Override
+ public int createTermHashes(List<GammaId> gammaIds) {
+ return getIndexer().createTermHashes(gammaIds);
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java
index 8267e07650a..71b185b0d5f 100644
--- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java
+++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java
@@ -27,9 +27,8 @@ import org.eclipse.osee.framework.core.data.AttributeTypeToken;
import org.eclipse.osee.framework.core.data.RelationTypeSide;
import org.eclipse.osee.framework.core.data.RelationTypeToken;
import org.eclipse.osee.framework.core.data.TransactionId;
-import org.eclipse.osee.framework.core.enums.QueryOption;
-import org.eclipse.osee.framework.core.data.RelationTypeId;
import org.eclipse.osee.framework.core.data.ValueKind;
+import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.core.enums.RelationSide;
import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.orcs.data.ArtifactReadable;
@@ -123,7 +122,7 @@ public interface QueryBuilder extends Query {
<R> ArtifactQuerySelection<R> selectInto(R receiver);
public <T> void selectAtt(AttributeTypeToken<T> attributeType, Consumer<T> consumer);
- public static AttributeTypeToken ANY_ATTRIBUTE_TYPE =
+ public static AttributeTypeToken<Object> ANY_ATTRIBUTE_TYPE =
AttributeTypeToken.valueOf(Long.MIN_VALUE, "Any Attribute Type");
QueryBuilder includeDeletedArtifacts();
@@ -324,6 +323,12 @@ public interface QueryBuilder extends Query {
QueryBuilder followNoSelect(RelationTypeSide relationTypeSide, ArtifactTypeToken artifacType);
+ QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive);
+
+ QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive);
+
+ QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive);
+
/**
* @deprecated use follow instead, currently still needed only for ORCS script
*/
diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java
index 1f6ee433d13..4dbcebc8c28 100644
--- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java
+++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java
@@ -18,7 +18,9 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.data.AttributeTypeGeneric;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
/**
@@ -32,7 +34,7 @@ public interface QueryIndexer {
Callable<List<Future<?>>> indexResources(Iterable<Long> gammaIds, IndexerCollector... collector);
- void indexAttrTypeIds(Iterable<Long> gammaIds);
+ void indexAttrTypeIds(Iterable<Long> attrTypeIds);
void indexMissingByAttrTypeIds(Iterable<Long> attrTypeIds);
@@ -42,4 +44,7 @@ public interface QueryIndexer {
CancellableCallable<Integer> purgeAllIndexes();
-}
+ int createTermHashes(Iterable<GammaId> gammaIds);
+
+ int createTermHashes(AttributeTypeId attributeType);
+} \ No newline at end of file

Back to the top