Skip to main content
aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRyan D. Brooks2018-02-27 14:15:51 +0000
committerRyan D. Brooks2019-08-22 22:02:36 +0000
commit50dc020baddb15a6ee3bc5f1eef4665d38dbb280 (patch)
tree44800ceeafdcc645fc4fbb5997484d6e690ee5c5
parent6cb208d3d8e8879985adb65de69b3f36ef7d3cbb (diff)
downloadorg.eclipse.osee-50dc020baddb15a6ee3bc5f1eef4665d38dbb280.tar.gz
org.eclipse.osee-50dc020baddb15a6ee3bc5f1eef4665d38dbb280.tar.xz
org.eclipse.osee-50dc020baddb15a6ee3bc5f1eef4665d38dbb280.zip
feature: Create faster query indexer
- Provides a faster and more flexible approach for indexing the - attribute values for various kinds of search including whole match, - tokenized by whitespace, and tokenized by non-alphanumeric. Any one - of the proceeding options can be combined with case sensitivity. Change-Id: I97077898e9827786c806460afbb98301c1d5ebbd
-rw-r--r--plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java8
-rw-r--r--plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java1
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java20
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java5
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java67
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java12
-rw-r--r--plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java14
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql6
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java216
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java153
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java2
-rw-r--r--plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java13
-rw-r--r--plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java11
-rw-r--r--plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java3
-rw-r--r--plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java13
-rw-r--r--plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java11
-rw-r--r--plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java9
17 files changed, 546 insertions, 18 deletions
diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java
index 5e076360cf2..1bf3fb915f0 100644
--- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java
+++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java
@@ -14,6 +14,7 @@ package org.eclipse.osee.framework.core.enums;
* @author John Misinco
*/
public enum QueryOption {
+ NONE,
CASE__MATCH,
CASE__IGNORE,
@@ -27,7 +28,12 @@ public enum QueryOption {
// matching the token order
TOKEN_MATCH_ORDER__ANY,
- TOKEN_MATCH_ORDER__MATCH;
+ TOKEN_MATCH_ORDER__MATCH,
+
+ // can be used to match one value or any values (i.e. using OR) but not all values (i.e. using AND)
+ WHOLE_MATCH,
+ TOKENIZE_WHITESPACE,
+ TOKENIZE_NON_ALPHANUMERIC;
public static QueryOption getTokenOrderType(boolean matchOrder) {
return matchOrder ? TOKEN_MATCH_ORDER__MATCH : TOKEN_MATCH_ORDER__ANY;
diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java
index 891958967cb..4de4464acd1 100644
--- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java
+++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java
@@ -19,6 +19,7 @@ public enum TableEnum {
BRANCH_TABLE("osee_branch", "br", ObjectType.BRANCH),
CHAR_JOIN_TABLE("osee_join_char_id", "jch"),
ID_JOIN_TABLE("osee_join_id", "jid"),
+ SEARCH_HASH_TABLE("osee_search_hash", "hsh"),
JOIN_ID4_TABLE("osee_join_id4", "jart"),
MERGE_TABLE("osee_merge", "mbr"),
RELATION_TABLE("osee_relation_link", "rel", ObjectType.RELATION),
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java
index 304563875a4..1b4c4b89128 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java
@@ -52,6 +52,7 @@ import org.eclipse.osee.orcs.QueryType;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactGuids;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType;
+import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists;
@@ -292,8 +293,10 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch {
return branch;
}
- public void select(AttributeTypeId attributeType) {
+ @Override
+ public QueryBuilder select(AttributeTypeId attributeType) {
this.attributeType = attributeType;
+ return this;
}
@Override
@@ -501,6 +504,21 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch {
}
@Override
+ public QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) {
+ return addAndCheck(new CriteriaAttributeHash(attributeType, value, option, caseSensitive));
+ }
+
+ @Override
+ public QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) {
+ return addAndCheck(new CriteriaAttributeHash(attributeType, values, option, caseSensitive));
+ }
+
+ @Override
+ public QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive) {
+ return addAndCheck(new CriteriaAttributeHash(value, option, caseSensitive));
+ }
+
+ @Override
public QueryBuilder and(AttributeTypeId attributeType, String value, QueryOption... options) {
return and(Collections.singleton(attributeType), Collections.singleton(value), options);
}
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java
index 943977de96e..71298570f43 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java
@@ -16,6 +16,7 @@ import java.util.Set;
import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
import org.eclipse.osee.orcs.OrcsSession;
import org.eclipse.osee.orcs.data.AttributeTypes;
@@ -45,4 +46,8 @@ public interface QueryEngineIndexer {
void indexAttrTypeIds(OrcsSession session, AttributeTypes attributeTypes, Iterable<Long> attrTypeIds);
void indexAttrTypeMissingOnly(AttributeTypes attributeTypes, Iterable<Long> attrTypeIds);
+
+ int createTermHashes(Iterable<GammaId> gammaIds);
+
+ int createTermHashes(AttributeTypeId attributeType);
} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java
new file mode 100644
index 00000000000..b960a673ff3
--- /dev/null
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java
@@ -0,0 +1,67 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Boeing.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Boeing - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.osee.orcs.core.ds.criteria;
+
+import java.util.Collection;
+import java.util.Collections;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
+import org.eclipse.osee.framework.core.enums.QueryOption;
+import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException;
+import org.eclipse.osee.orcs.core.ds.Criteria;
+import org.eclipse.osee.orcs.core.ds.Options;
+
+/**
+ * @author Ryan D. Brooks
+ */
+public class CriteriaAttributeHash extends Criteria {
+ private final AttributeTypeId attributeType;
+ private final Collection<String> values;
+ private final QueryOption option;
+ private final boolean caseSensitive;
+
+ public CriteriaAttributeHash(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) {
+ this.attributeType = attributeType;
+ this.values = values;
+ this.option = option;
+ this.caseSensitive = caseSensitive;
+ }
+
+ public CriteriaAttributeHash(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) {
+ this(attributeType, Collections.singletonList(value), option, caseSensitive);
+ }
+
+ public CriteriaAttributeHash(String value, QueryOption option, boolean caseSensitive) {
+ this(AttributeTypeId.SENTINEL, value, option, caseSensitive);
+ }
+
+ public AttributeTypeId getAttributeType() {
+ return attributeType;
+ }
+
+ public Collection<String> getValues() {
+ return values;
+ }
+
+ public QueryOption getOption() {
+ return option;
+ }
+
+ public boolean isCaseSensitive() {
+ return caseSensitive;
+ }
+
+ @Override
+ public void checkValid(Options options) {
+ if (values.isEmpty()) {
+ throw new OseeArgumentException("CriteriaAttributeHash requires at least one value to match");
+ }
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java
index f61a361bee3..2364a220a4e 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java
@@ -16,6 +16,7 @@ import java.util.Set;
import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
import org.eclipse.osee.orcs.OrcsSession;
import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer;
@@ -88,4 +89,13 @@ public class QueryIndexerImpl implements QueryIndexer {
return engineIndexer.purgeAllIndexes(session);
}
-}
+ @Override
+ public int createTermHashes(Iterable<GammaId> gammaIds) {
+ return engineIndexer.createTermHashes(gammaIds);
+ }
+
+ @Override
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return engineIndexer.createTermHashes(attributeType);
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java
index 08c1e7317c4..2ab989f95ae 100644
--- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java
+++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java
@@ -38,11 +38,11 @@ import org.eclipse.osee.framework.core.enums.CoreArtifactTokens;
import org.eclipse.osee.framework.core.enums.CoreArtifactTypes;
import org.eclipse.osee.framework.core.enums.CoreAttributeTypes;
import org.eclipse.osee.framework.core.enums.CoreTupleTypes;
+import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.core.enums.RelationSorter;
import org.eclipse.osee.framework.core.enums.TableEnum;
import org.eclipse.osee.framework.jdk.core.type.Id;
import org.eclipse.osee.framework.jdk.core.type.OseeCoreException;
-import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.framework.jdk.core.util.Conditions;
import org.eclipse.osee.orcs.KeyValueOps;
import org.eclipse.osee.orcs.OrcsApi;
@@ -152,12 +152,12 @@ public class TransactionBuilderImpl implements TransactionBuilder {
@Override
public List<ArtifactToken> createArtifacts(ArtifactTypeId artifactType, ArtifactId parent, List<String> names) {
- ResultSet<ArtifactReadable> results =
- queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).and(CoreAttributeTypes.Name,
- names).getResults();
- if (!results.isEmpty()) {
- throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", results.size(),
- artifactType, results.getList());
+ List<ArtifactToken> duplicates =
+ queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).andAttribute(CoreAttributeTypes.Name, names,
+ QueryOption.TOKENIZE_WHITESPACE, false).loadArtifactTokens();
+ if (!duplicates.isEmpty()) {
+ throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", duplicates.size(),
+ artifactType, duplicates);
}
List<ArtifactToken> tokens = new ArrayList<>(names.size());
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql
index c99f02b08f2..341e1255f35 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql
@@ -1,3 +1,9 @@
+CREATE TABLE OSEE_SEARCH_HASH (
+ app_id ${db.bigint} NOT NULL,
+ hash ${db.bigint} NOT NULL,
+ gamma_id ${db.bigint} NOT NULL,
+ CONSTRAINT SEARCH_HASH__A_H_G_PK PRIMARY KEY (app_id, hash, gamma_id))
+ ${db.organization_index};
-- OSEE_ARTIFACT
CREATE TABLE OSEE_ARTIFACT (
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java
new file mode 100644
index 00000000000..337daab8f0a
--- /dev/null
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java
@@ -0,0 +1,216 @@
+/*******************************************************************************
+ * Copyright (c) 218 Boeing.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Boeing - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.osee.orcs.db.internal.search;
+
+import static org.eclipse.osee.jdbc.JdbcConstants.JDBC__MAX_FETCH_SIZE;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
+import org.eclipse.osee.framework.core.data.GammaId;
+import org.eclipse.osee.framework.core.enums.QueryOption;
+import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException;
+import org.eclipse.osee.jdbc.JdbcClient;
+import org.eclipse.osee.jdbc.JdbcStatement;
+import org.eclipse.osee.jdbc.OseePreparedStatement;
+import org.eclipse.osee.orcs.db.internal.sql.join.IdJoinQuery;
+import org.eclipse.osee.orcs.db.internal.sql.join.SqlJoinFactory;
+
+/**
+ * Instances of this class should not be shared among threads. Use multiple term normalization rules and then store the
+ * unique set of resulting hashes. Each normalization starts with making all letters lower-case.</br>
+ * <li>whole match = no tokenize; keep only alpha-numeric characters
+ * <li>original = tokenize on characters that are not digits or letters
+ * <li>partial with fuzzy punc = tokenize on whitespace; strip all punctuation
+ * <li>maybe not = tokenize on whitespace; strip leading and trailing punctuation; replace remaining punctuation with
+ * spaces
+ * <ul>
+ * <li>{Requirement_Name}
+ * <li>use new tablespace called osee_search Replace TagProcessor and use XmlTextInputStream
+ *
+ * @author Ryan D. Brooks
+ */
+public class SearchTermHash {
+ private static final String SELECT_MISSING_ATTRIBUTES =
+ "SELECT gamma_id, value FROM osee_attribute att WHERE attr_type_id = ? AND value is not null AND not exists (SELECT 1 FROM osee_search_hash has WHERE has.gamma_id = att.gamma_id)";
+
+ private static final String SELECT_ATTRIBUTES_BY_TYPE =
+ "SELECT gamma_id, value FROM osee_attribute WHERE attr_type_id = ? AND value is not null";
+
+ private static final String SELECT_ATTRIBUTES_BY_GAMMA =
+ "SELECT att.gamma_id, value FROM osee_attribute att, osee_join_id WHERE query_id = ? AND id = gamma_id AND value is not null";
+
+ private final SqlJoinFactory joinFactory;
+ private final JdbcClient jdbcClient;
+ private OseePreparedStatement insertStatement;
+ private final HashSet<Long> hashes = new HashSet<>();
+ private final List<List<Long>> resultHashes = new ArrayList<>();
+ private final List<Long> singleTokens = new ArrayList<>();
+
+ public SearchTermHash() {
+ this(null, null);
+ }
+
+ public SearchTermHash(JdbcClient jdbcClient, SqlJoinFactory joinFactory) {
+ this.jdbcClient = jdbcClient;
+ this.joinFactory = joinFactory;
+ }
+
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return createTermHashes(SELECT_ATTRIBUTES_BY_TYPE, attributeType);
+ }
+
+ public int createTermHashes(Iterable<GammaId> gammaIds) {
+ try (IdJoinQuery joinQuery = joinFactory.createIdJoinQuery()) {
+ joinQuery.addAndStore(gammaIds);
+ return createTermHashes(SELECT_ATTRIBUTES_BY_GAMMA, joinQuery.getQueryId());
+ }
+ }
+
+ private int createTermHashes(String sql, Object... data) {
+ insertStatement =
+ jdbcClient.getBatchStatement("INSERT INTO osee_search_hash (app_id, hash, gamma_id) VALUES (?, ?, ?)");
+ int loaded = jdbcClient.runQuery(this::createTermHashes, JDBC__MAX_FETCH_SIZE, sql, data);
+ insertStatement.execute();
+ return loaded;
+ }
+
+ public List<List<Long>> getTermHashes(Collection<String> values, QueryOption queryOption) {
+ resultHashes.clear();
+ singleTokens.clear();
+ resultHashes.add(singleTokens);
+ for (String value : values) {
+ switch (queryOption) {
+ case TOKENIZE_NON_ALPHANUMERIC:
+ addOriginalMatchHash(value);
+ break;
+ case TOKENIZE_WHITESPACE:
+ addPartialFuzzyPuncHash(value);
+ break;
+ case WHOLE_MATCH:
+ getWholeMatchHash(value);
+ break;
+ default:
+ throw new OseeArgumentException("Unexpected query option %s", queryOption);
+ }
+ copyHashes();
+ }
+ return resultHashes;
+ }
+
+ private void copyHashes() {
+ if (hashes.size() == 1) {
+ singleTokens.add(hashes.iterator().next());
+ } else {
+ resultHashes.add(new ArrayList<>(hashes));
+ }
+ hashes.clear();
+ }
+
+ private void createTermHashes(JdbcStatement stmt) {
+ String value = stmt.getString("value");
+ addOriginalMatchHash(value);
+ addPartialFuzzyPuncHash(value);
+ getWholeMatchHash(value);
+
+ for (Long hash : hashes) {
+ insertStatement.addToBatch(1, hash, stmt.getLong("gamma_id"));
+ }
+ hashes.clear();
+ }
+
+ /**
+ * don't tokenize and include only alpha-numeric characters in hash. If the value contains only non alpha-numeric
+ * characters, then the hash is computed using all of its characters.
+ */
+ public void getWholeMatchHash(String value) {
+ long hash = 0;
+ for (char c : value.toCharArray()) {
+ c = toLower(c);
+ if (isLetterOrDigit(c)) {
+ hash = 31 * hash + c;
+ }
+ }
+
+ hashes.add(handleNoAlphaNumeric(value, hash));
+ }
+
+ private static long handleNoAlphaNumeric(String value, long hash) {
+ if (hash == 0) {
+ for (char c : value.toCharArray()) {
+ hash = 31 * hash + toLower(c);
+ }
+ }
+ return hash;
+ }
+
+ /**
+ * tokenize on everything except alpha-numeric characters
+ */
+ private void addOriginalMatchHash(String value) {
+ long hash = 0;
+ for (char c : value.toCharArray()) {
+ c = toLower(c);
+ if (isLetterOrDigit(c)) {
+ hash = 31 * hash + c;
+ } else {
+ addHash(hash);
+ hash = 0;
+ }
+ }
+ addHash(hash);
+ }
+
+ /**
+ * tokenize on whitespace and include only alpha-numeric characters in hash
+ */
+ private void addPartialFuzzyPuncHash(String value) {
+ long hash = 0;
+ for (char c : value.toCharArray()) {
+ if (c == ' ' || c == '\n' || c == '\r' || c == '\t') {
+ addHash(hash);
+ hash = 0;
+ } else {
+ c = toLower(c);
+ if (isLetterOrDigit(c)) {
+ hash = 31 * hash + c;
+ }
+ }
+ }
+ addHash(hash);
+ }
+
+ private void addHash(Long hash) {
+ if (hash != 0) {
+ hashes.add(hash);
+ }
+ }
+
+ private static boolean isLetterOrDigit(char c) {
+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z');
+ }
+
+ private static char toLower(char c) {
+ if (c >= 'A' && c <= 'Z') {
+ return (char) (c + 32);
+ }
+ return c;
+ }
+
+ public static void main(String[] args) {
+ SearchTermHash app = new SearchTermHash();
+ System.out.println(app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.WHOLE_MATCH));
+ System.out.println(
+ app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.TOKENIZE_WHITESPACE));
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java
new file mode 100644
index 00000000000..1c74ec832fe
--- /dev/null
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java
@@ -0,0 +1,153 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Boeing.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Boeing - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.osee.orcs.db.internal.search.handlers;
+
+import java.util.Iterator;
+import java.util.List;
+import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash;
+import org.eclipse.osee.orcs.db.internal.search.SearchTermHash;
+import org.eclipse.osee.orcs.db.internal.sql.AbstractSqlWriter;
+import org.eclipse.osee.orcs.db.internal.sql.SqlHandler;
+import org.eclipse.osee.orcs.db.internal.sql.TableEnum;
+
+/**
+ * @author Ryan D. Brooks
+ */
+public class AttributeHashQueryHandler extends SqlHandler<CriteriaAttributeHash> {
+ private String attTxsAlias;
+ private String attAlias;
+ private String artAlias;
+ private CriteriaAttributeHash criteria;
+ private final SearchTermHash hasher = new SearchTermHash();
+ private List<List<Long>> hashes;
+
+ @Override
+ public void setData(CriteriaAttributeHash criteria) {
+ this.criteria = criteria;
+ }
+
+ @Override
+ public void addTables(AbstractSqlWriter writer) {
+ hashes = hasher.getTermHashes(criteria.getValues(), criteria.getOption());
+ int hashTableCount = countNeededHashTables();
+
+ for (int i = 0; i < hashTableCount; i++) {
+ writer.addTable(TableEnum.SEARCH_HASH_TABLE);
+ }
+ attAlias = writer.addTable(TableEnum.ATTRIBUTE_TABLE);
+ artAlias = writer.getMainTableAlias(TableEnum.ARTIFACT_TABLE);
+ attTxsAlias = writer.addTable(TableEnum.TXS_TABLE);
+ }
+
+ private int countNeededHashTables() {
+ int hashTableCount = 0;
+ Iterator<List<Long>> iterator = hashes.iterator();
+ List<Long> singleTokens = iterator.next();
+ if (!singleTokens.isEmpty()) {
+ hashTableCount++;
+ }
+ while (iterator.hasNext()) {
+ List<Long> list = iterator.next();
+ hashTableCount += list.size();
+ }
+ return hashTableCount;
+ }
+
+ /**
+ * Combine all single tokens together using a simple in clause
+ */
+ private void writeHashTableForSingleTokens(AbstractSqlWriter writer, String hshAlias, List<Long> singleTokens) {
+ writer.write("(");
+ writer.writeEqualsParameterAnd(hshAlias, "app_id", 1);
+ if (singleTokens.size() == 1) {
+ writer.writeEqualsParameter(hshAlias, "hash", singleTokens.get(0));
+ } else {
+ writer.write(hshAlias);
+ writer.write(".hash IN (");
+ boolean notFirst = false;
+ for (Long hash : singleTokens) {
+ if (notFirst) {
+ writer.write(", ?");
+ } else {
+ writer.write("?");
+ notFirst = true;
+ }
+ writer.addParameter(hash);
+ }
+ writer.write(")");
+ }
+ writer.write(" AND ");
+ writer.writeEquals(hshAlias, attAlias, "gamma_id");
+ writer.write(")");
+ }
+
+ private void addHashPredicates(AbstractSqlWriter writer) {
+ Iterator<String> aliases = writer.getAliases(TableEnum.SEARCH_HASH_TABLE).iterator();
+ Iterator<List<Long>> hashIterator = hashes.iterator();
+ String prevHshAlias = aliases.next();
+ List<Long> singleTokens = hashIterator.next();
+
+ writer.write("(");
+ boolean hasSingleTokens = !singleTokens.isEmpty();
+ if (hasSingleTokens) {
+ writeHashTableForSingleTokens(writer, prevHshAlias, singleTokens);
+ }
+
+ boolean useOr = hasSingleTokens;
+ while (hashIterator.hasNext()) {
+ if (useOr) {
+ writer.write(" OR\n");
+ } else {
+ useOr = true;
+ }
+ writer.write(" (");
+
+ boolean joinGammaIds = false;
+ for (Long hash : hashIterator.next()) {
+ String hshAlias = aliases.next();
+ if (joinGammaIds) {
+ writer.writeEquals(prevHshAlias, hshAlias, "gamma_id");
+ writer.write(" AND ");
+ } else {
+ joinGammaIds = true;
+ }
+ writer.write(hshAlias);
+ writer.write(".app_id = ? AND ");
+ writer.addParameter(1);
+ writer.write(hshAlias);
+ writer.write(".hash = ? AND ");
+ writer.addParameter(hash);
+ prevHshAlias = hshAlias;
+ }
+ writer.writeEquals(prevHshAlias, attAlias, "gamma_id");
+ writer.write(")");
+ }
+ writer.write(") AND\n ");
+ }
+
+ @Override
+ public void addPredicates(AbstractSqlWriter writer) {
+ addHashPredicates(writer);
+ if (criteria.getAttributeType().isValid()) {
+ writer.writeEqualsParameterAnd(attAlias, "attr_type_id", criteria.getAttributeType());
+ }
+ writer.writeEquals(attAlias, artAlias, "art_id");
+ writer.write(" AND ");
+ writer.writeEquals(attAlias, attTxsAlias, "gamma_id");
+ writer.write(" AND ");
+ writer.write(writer.getTxBranchFilter(attTxsAlias));
+ }
+
+ @Override
+ public int getPriority() {
+ return SqlHandlerPriority.ATTRIBUTE_TOKENIZED_VALUE.ordinal();
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java
index 11fb853958c..706e4187484 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java
@@ -20,6 +20,7 @@ import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactGuids;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAssociatedArtId;
+import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw;
import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists;
@@ -93,6 +94,7 @@ public final class SqlHandlerFactoryUtil {
}
private static void addArtifactHandlers(Map<Class<? extends Criteria>, Class<? extends SqlHandler<?>>> handleMap) {
+ handleMap.put(CriteriaAttributeHash.class, AttributeHashQueryHandler.class);
handleMap.put(CriteriaArtifactGuids.class, ArtifactGuidSqlHandler.class);
handleMap.put(CriteriaArtifactIds.class, ArtifactIdsSqlHandler.class);
handleMap.put(CriteriaArtifactType.class, ArtifactTypeSqlHandler.class);
diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java
index 4d8e0baf1b7..44881656ccc 100644
--- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java
+++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java
@@ -17,6 +17,7 @@ import java.util.Set;
import java.util.concurrent.Future;
import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
import org.eclipse.osee.framework.jdk.core.type.OseeCoreException;
import org.eclipse.osee.framework.jdk.core.util.Collections;
@@ -28,6 +29,7 @@ import org.eclipse.osee.orcs.OrcsSession;
import org.eclipse.osee.orcs.core.ds.IndexerData;
import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer;
import org.eclipse.osee.orcs.data.AttributeTypes;
+import org.eclipse.osee.orcs.db.internal.search.SearchTermHash;
import org.eclipse.osee.orcs.db.internal.search.indexer.callable.DeleteTagSetDatabaseTxCallable;
import org.eclipse.osee.orcs.db.internal.search.indexer.callable.IndexerDatabaseStatisticsCallable;
import org.eclipse.osee.orcs.db.internal.search.indexer.callable.PurgeAllTagsDatabaseCallable;
@@ -155,4 +157,13 @@ public class QueryEngineIndexerImpl implements QueryEngineIndexer {
systemCollector.removeCollector(collector);
}
-}
+ @Override
+ public int createTermHashes(Iterable<GammaId> gammaIds) {
+ return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(gammaIds);
+ }
+
+ @Override
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(attributeType);
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java
index 88534a8a0c2..a7ae5d1635a 100644
--- a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java
+++ b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java
@@ -10,15 +10,19 @@
*******************************************************************************/
package org.eclipse.osee.orcs.rest.model;
+import java.util.List;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
+import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
+import org.eclipse.osee.framework.core.data.GammaId;
/**
* @author Roberto E. Escobar
@@ -36,6 +40,13 @@ public interface IndexerEndpoint {
@Consumes(MediaType.APPLICATION_JSON)
Response indexResources(IndexResources options);
+ @POST
+ @Path("attributeType/{attributeType}")
+ int createTermHashes(@PathParam("attributeType") AttributeTypeId attributeType);
+
+ @POST
+ int createTermHashes(List<GammaId> gammaIds);
+
@DELETE
@Path("queue")
Response deleteIndexQueue();
diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java
index d42dea127c4..fa024331276 100644
--- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java
+++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java
@@ -25,6 +25,7 @@ import org.eclipse.osee.framework.core.data.BranchId;
import org.eclipse.osee.framework.core.data.TransactionId;
import org.eclipse.osee.framework.core.data.TransactionToken;
import org.eclipse.osee.framework.core.data.UserId;
+import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.jdk.core.type.MatchLocation;
import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.orcs.OrcsApi;
@@ -148,7 +149,7 @@ public class ArtifactEndpointImpl implements ArtifactEndpoint {
}
if (attributeType.isValid()) {
if (exists) {
- query.andAttributeIs(attributeType, value);
+ query.andAttribute(attributeType, value, QueryOption.WHOLE_MATCH, false);
} else {
query.andNotExists(attributeType, value);
}
diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java
index 5e4a960233d..96eebcb30db 100644
--- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java
+++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java
@@ -19,8 +19,10 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
import org.eclipse.osee.framework.core.data.BranchId;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.framework.jdk.core.util.Collections;
import org.eclipse.osee.jaxrs.OseeWebApplicationException;
@@ -101,4 +103,13 @@ public class IndexerEndpointImpl implements IndexerEndpoint {
return asResponse(modified);
}
-}
+ @Override
+ public int createTermHashes(AttributeTypeId attributeType) {
+ return getIndexer().createTermHashes(attributeType);
+ }
+
+ @Override
+ public int createTermHashes(List<GammaId> gammaIds) {
+ return getIndexer().createTermHashes(gammaIds);
+ }
+} \ No newline at end of file
diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java
index d8a1e084afb..dc7321c705a 100644
--- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java
+++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java
@@ -25,10 +25,9 @@ import org.eclipse.osee.framework.core.data.IRelationType;
import org.eclipse.osee.framework.core.data.RelationTypeId;
import org.eclipse.osee.framework.core.data.RelationTypeSide;
import org.eclipse.osee.framework.core.data.TransactionId;
-import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.core.enums.RelationSide;
-import org.eclipse.osee.framework.core.data.RelationTypeId;
import org.eclipse.osee.framework.core.data.ValueKind;
+import org.eclipse.osee.framework.core.enums.QueryOption;
import org.eclipse.osee.framework.core.enums.RelationSide;
import org.eclipse.osee.framework.jdk.core.type.ResultSet;
import org.eclipse.osee.orcs.data.ArtifactReadable;
@@ -122,7 +121,7 @@ public interface QueryBuilder extends Query {
<R> ArtifactQuerySelection<R> selectInto(R receiver);
public <T> void selectAtt(AttributeTypeToken<T> attributeType, Consumer<T> consumer);
- public static AttributeTypeToken ANY_ATTRIBUTE_TYPE =
+ public static AttributeTypeToken<Object> ANY_ATTRIBUTE_TYPE =
AttributeTypeToken.valueOf(Long.MIN_VALUE, "Any Attribute Type");
QueryBuilder includeDeletedArtifacts();
@@ -317,6 +316,12 @@ public interface QueryBuilder extends Query {
QueryBuilder followNoSelect(RelationTypeSide relationTypeSide, ArtifactTypeToken artifacType);
+ QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive);
+
+ QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive);
+
+ QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive);
+
/**
* @deprecated use follow instead, currently still needed only for ORCS script
*/
diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java
index dc857765714..bc7b81fe220 100644
--- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java
+++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java
@@ -14,7 +14,9 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
+import org.eclipse.osee.framework.core.data.AttributeTypeId;
import org.eclipse.osee.framework.core.data.Branch;
+import org.eclipse.osee.framework.core.data.GammaId;
import org.eclipse.osee.framework.core.executor.CancellableCallable;
/**
@@ -28,7 +30,7 @@ public interface QueryIndexer {
Callable<List<Future<?>>> indexResources(Iterable<Long> gammaIds, IndexerCollector... collector);
- void indexAttrTypeIds(Iterable<Long> gammaIds);
+ void indexAttrTypeIds(Iterable<Long> attrTypeIds);
void indexMissingByAttrTypeIds(Iterable<Long> attrTypeIds);
@@ -38,4 +40,7 @@ public interface QueryIndexer {
CancellableCallable<Integer> purgeAllIndexes();
-}
+ int createTermHashes(Iterable<GammaId> gammaIds);
+
+ int createTermHashes(AttributeTypeId attributeType);
+} \ No newline at end of file

Back to the top