diff options
author | Ryan D. Brooks | 2018-02-27 14:15:51 +0000 |
---|---|---|
committer | Ryan D. Brooks | 2019-08-22 22:02:36 +0000 |
commit | 50dc020baddb15a6ee3bc5f1eef4665d38dbb280 (patch) | |
tree | 44800ceeafdcc645fc4fbb5997484d6e690ee5c5 | |
parent | 6cb208d3d8e8879985adb65de69b3f36ef7d3cbb (diff) | |
download | org.eclipse.osee-50dc020baddb15a6ee3bc5f1eef4665d38dbb280.tar.gz org.eclipse.osee-50dc020baddb15a6ee3bc5f1eef4665d38dbb280.tar.xz org.eclipse.osee-50dc020baddb15a6ee3bc5f1eef4665d38dbb280.zip |
feature: Create faster query indexer
- Provides a faster and more flexible approach for indexing the
- attribute values for various kinds of search including whole match,
- tokenized by whitespace, and tokenized by non-alphanumeric. Any one
- of the proceeding options can be combined with case sensitivity.
Change-Id: I97077898e9827786c806460afbb98301c1d5ebbd
17 files changed, 546 insertions, 18 deletions
diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java index 5e076360cf2..1bf3fb915f0 100644 --- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java +++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java @@ -14,6 +14,7 @@ package org.eclipse.osee.framework.core.enums; * @author John Misinco */ public enum QueryOption { + NONE, CASE__MATCH, CASE__IGNORE, @@ -27,7 +28,12 @@ public enum QueryOption { // matching the token order TOKEN_MATCH_ORDER__ANY, - TOKEN_MATCH_ORDER__MATCH; + TOKEN_MATCH_ORDER__MATCH, + + // can be used to match one value or any values (i.e. using OR) but not all values (i.e. using AND) + WHOLE_MATCH, + TOKENIZE_WHITESPACE, + TOKENIZE_NON_ALPHANUMERIC; public static QueryOption getTokenOrderType(boolean matchOrder) { return matchOrder ? TOKEN_MATCH_ORDER__MATCH : TOKEN_MATCH_ORDER__ANY; diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java index 891958967cb..4de4464acd1 100644 --- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java +++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/TableEnum.java @@ -19,6 +19,7 @@ public enum TableEnum { BRANCH_TABLE("osee_branch", "br", ObjectType.BRANCH), CHAR_JOIN_TABLE("osee_join_char_id", "jch"), ID_JOIN_TABLE("osee_join_id", "jid"), + SEARCH_HASH_TABLE("osee_search_hash", "hsh"), JOIN_ID4_TABLE("osee_join_id4", "jart"), MERGE_TABLE("osee_merge", "mbr"), RELATION_TABLE("osee_relation_link", "rel", ObjectType.RELATION), diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java index 304563875a4..1b4c4b89128 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java @@ -52,6 +52,7 @@ import org.eclipse.osee.orcs.QueryType; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactGuids; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType; +import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists; @@ -292,8 +293,10 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch { return branch; } - public void select(AttributeTypeId attributeType) { + @Override + public QueryBuilder select(AttributeTypeId attributeType) { this.attributeType = attributeType; + return this; } @Override @@ -501,6 +504,21 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch { } @Override + public QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) { + return addAndCheck(new CriteriaAttributeHash(attributeType, value, option, caseSensitive)); + } + + @Override + public QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) { + return addAndCheck(new CriteriaAttributeHash(attributeType, values, option, caseSensitive)); + } + + @Override + public QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive) { + return addAndCheck(new CriteriaAttributeHash(value, option, caseSensitive)); + } + + @Override public QueryBuilder and(AttributeTypeId attributeType, String value, QueryOption... options) { return and(Collections.singleton(attributeType), Collections.singleton(value), options); } diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java index 943977de96e..71298570f43 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java @@ -16,6 +16,7 @@ import java.util.Set; import java.util.concurrent.Future; import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; import org.eclipse.osee.orcs.OrcsSession; import org.eclipse.osee.orcs.data.AttributeTypes; @@ -45,4 +46,8 @@ public interface QueryEngineIndexer { void indexAttrTypeIds(OrcsSession session, AttributeTypes attributeTypes, Iterable<Long> attrTypeIds); void indexAttrTypeMissingOnly(AttributeTypes attributeTypes, Iterable<Long> attrTypeIds); + + int createTermHashes(Iterable<GammaId> gammaIds); + + int createTermHashes(AttributeTypeId attributeType); }
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java new file mode 100644 index 00000000000..b960a673ff3 --- /dev/null +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2018 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.orcs.core.ds.criteria; + +import java.util.Collection; +import java.util.Collections; +import org.eclipse.osee.framework.core.data.AttributeTypeId; +import org.eclipse.osee.framework.core.enums.QueryOption; +import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; +import org.eclipse.osee.orcs.core.ds.Criteria; +import org.eclipse.osee.orcs.core.ds.Options; + +/** + * @author Ryan D. Brooks + */ +public class CriteriaAttributeHash extends Criteria { + private final AttributeTypeId attributeType; + private final Collection<String> values; + private final QueryOption option; + private final boolean caseSensitive; + + public CriteriaAttributeHash(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) { + this.attributeType = attributeType; + this.values = values; + this.option = option; + this.caseSensitive = caseSensitive; + } + + public CriteriaAttributeHash(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) { + this(attributeType, Collections.singletonList(value), option, caseSensitive); + } + + public CriteriaAttributeHash(String value, QueryOption option, boolean caseSensitive) { + this(AttributeTypeId.SENTINEL, value, option, caseSensitive); + } + + public AttributeTypeId getAttributeType() { + return attributeType; + } + + public Collection<String> getValues() { + return values; + } + + public QueryOption getOption() { + return option; + } + + public boolean isCaseSensitive() { + return caseSensitive; + } + + @Override + public void checkValid(Options options) { + if (values.isEmpty()) { + throw new OseeArgumentException("CriteriaAttributeHash requires at least one value to match"); + } + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java index f61a361bee3..2364a220a4e 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java @@ -16,6 +16,7 @@ import java.util.Set; import java.util.concurrent.Future; import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; import org.eclipse.osee.orcs.OrcsSession; import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer; @@ -88,4 +89,13 @@ public class QueryIndexerImpl implements QueryIndexer { return engineIndexer.purgeAllIndexes(session); } -} + @Override + public int createTermHashes(Iterable<GammaId> gammaIds) { + return engineIndexer.createTermHashes(gammaIds); + } + + @Override + public int createTermHashes(AttributeTypeId attributeType) { + return engineIndexer.createTermHashes(attributeType); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java index 08c1e7317c4..2ab989f95ae 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java @@ -38,11 +38,11 @@ import org.eclipse.osee.framework.core.enums.CoreArtifactTokens; import org.eclipse.osee.framework.core.enums.CoreArtifactTypes; import org.eclipse.osee.framework.core.enums.CoreAttributeTypes; import org.eclipse.osee.framework.core.enums.CoreTupleTypes; +import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.core.enums.RelationSorter; import org.eclipse.osee.framework.core.enums.TableEnum; import org.eclipse.osee.framework.jdk.core.type.Id; import org.eclipse.osee.framework.jdk.core.type.OseeCoreException; -import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.framework.jdk.core.util.Conditions; import org.eclipse.osee.orcs.KeyValueOps; import org.eclipse.osee.orcs.OrcsApi; @@ -152,12 +152,12 @@ public class TransactionBuilderImpl implements TransactionBuilder { @Override public List<ArtifactToken> createArtifacts(ArtifactTypeId artifactType, ArtifactId parent, List<String> names) { - ResultSet<ArtifactReadable> results = - queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).and(CoreAttributeTypes.Name, - names).getResults(); - if (!results.isEmpty()) { - throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", results.size(), - artifactType, results.getList()); + List<ArtifactToken> duplicates = + queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).andAttribute(CoreAttributeTypes.Name, names, + QueryOption.TOKENIZE_WHITESPACE, false).loadArtifactTokens(); + if (!duplicates.isEmpty()) { + throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", duplicates.size(), + artifactType, duplicates); } List<ArtifactToken> tokens = new ArrayList<>(names.size()); diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql index c99f02b08f2..341e1255f35 100644 --- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/resource/migration/V0.21.0_2015_02_25_1244__Osee_Versioning_Schema.sql @@ -1,3 +1,9 @@ +CREATE TABLE OSEE_SEARCH_HASH ( + app_id ${db.bigint} NOT NULL, + hash ${db.bigint} NOT NULL, + gamma_id ${db.bigint} NOT NULL, + CONSTRAINT SEARCH_HASH__A_H_G_PK PRIMARY KEY (app_id, hash, gamma_id)) + ${db.organization_index}; -- OSEE_ARTIFACT CREATE TABLE OSEE_ARTIFACT ( diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java new file mode 100644 index 00000000000..337daab8f0a --- /dev/null +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java @@ -0,0 +1,216 @@ +/******************************************************************************* + * Copyright (c) 218 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.orcs.db.internal.search; + +import static org.eclipse.osee.jdbc.JdbcConstants.JDBC__MAX_FETCH_SIZE; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import org.eclipse.osee.framework.core.data.AttributeTypeId; +import org.eclipse.osee.framework.core.data.GammaId; +import org.eclipse.osee.framework.core.enums.QueryOption; +import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; +import org.eclipse.osee.jdbc.JdbcClient; +import org.eclipse.osee.jdbc.JdbcStatement; +import org.eclipse.osee.jdbc.OseePreparedStatement; +import org.eclipse.osee.orcs.db.internal.sql.join.IdJoinQuery; +import org.eclipse.osee.orcs.db.internal.sql.join.SqlJoinFactory; + +/** + * Instances of this class should not be shared among threads. Use multiple term normalization rules and then store the + * unique set of resulting hashes. Each normalization starts with making all letters lower-case.</br> + * <li>whole match = no tokenize; keep only alpha-numeric characters + * <li>original = tokenize on characters that are not digits or letters + * <li>partial with fuzzy punc = tokenize on whitespace; strip all punctuation + * <li>maybe not = tokenize on whitespace; strip leading and trailing punctuation; replace remaining punctuation with + * spaces + * <ul> + * <li>{Requirement_Name} + * <li>use new tablespace called osee_search Replace TagProcessor and use XmlTextInputStream + * + * @author Ryan D. Brooks + */ +public class SearchTermHash { + private static final String SELECT_MISSING_ATTRIBUTES = + "SELECT gamma_id, value FROM osee_attribute att WHERE attr_type_id = ? AND value is not null AND not exists (SELECT 1 FROM osee_search_hash has WHERE has.gamma_id = att.gamma_id)"; + + private static final String SELECT_ATTRIBUTES_BY_TYPE = + "SELECT gamma_id, value FROM osee_attribute WHERE attr_type_id = ? AND value is not null"; + + private static final String SELECT_ATTRIBUTES_BY_GAMMA = + "SELECT att.gamma_id, value FROM osee_attribute att, osee_join_id WHERE query_id = ? AND id = gamma_id AND value is not null"; + + private final SqlJoinFactory joinFactory; + private final JdbcClient jdbcClient; + private OseePreparedStatement insertStatement; + private final HashSet<Long> hashes = new HashSet<>(); + private final List<List<Long>> resultHashes = new ArrayList<>(); + private final List<Long> singleTokens = new ArrayList<>(); + + public SearchTermHash() { + this(null, null); + } + + public SearchTermHash(JdbcClient jdbcClient, SqlJoinFactory joinFactory) { + this.jdbcClient = jdbcClient; + this.joinFactory = joinFactory; + } + + public int createTermHashes(AttributeTypeId attributeType) { + return createTermHashes(SELECT_ATTRIBUTES_BY_TYPE, attributeType); + } + + public int createTermHashes(Iterable<GammaId> gammaIds) { + try (IdJoinQuery joinQuery = joinFactory.createIdJoinQuery()) { + joinQuery.addAndStore(gammaIds); + return createTermHashes(SELECT_ATTRIBUTES_BY_GAMMA, joinQuery.getQueryId()); + } + } + + private int createTermHashes(String sql, Object... data) { + insertStatement = + jdbcClient.getBatchStatement("INSERT INTO osee_search_hash (app_id, hash, gamma_id) VALUES (?, ?, ?)"); + int loaded = jdbcClient.runQuery(this::createTermHashes, JDBC__MAX_FETCH_SIZE, sql, data); + insertStatement.execute(); + return loaded; + } + + public List<List<Long>> getTermHashes(Collection<String> values, QueryOption queryOption) { + resultHashes.clear(); + singleTokens.clear(); + resultHashes.add(singleTokens); + for (String value : values) { + switch (queryOption) { + case TOKENIZE_NON_ALPHANUMERIC: + addOriginalMatchHash(value); + break; + case TOKENIZE_WHITESPACE: + addPartialFuzzyPuncHash(value); + break; + case WHOLE_MATCH: + getWholeMatchHash(value); + break; + default: + throw new OseeArgumentException("Unexpected query option %s", queryOption); + } + copyHashes(); + } + return resultHashes; + } + + private void copyHashes() { + if (hashes.size() == 1) { + singleTokens.add(hashes.iterator().next()); + } else { + resultHashes.add(new ArrayList<>(hashes)); + } + hashes.clear(); + } + + private void createTermHashes(JdbcStatement stmt) { + String value = stmt.getString("value"); + addOriginalMatchHash(value); + addPartialFuzzyPuncHash(value); + getWholeMatchHash(value); + + for (Long hash : hashes) { + insertStatement.addToBatch(1, hash, stmt.getLong("gamma_id")); + } + hashes.clear(); + } + + /** + * don't tokenize and include only alpha-numeric characters in hash. If the value contains only non alpha-numeric + * characters, then the hash is computed using all of its characters. + */ + public void getWholeMatchHash(String value) { + long hash = 0; + for (char c : value.toCharArray()) { + c = toLower(c); + if (isLetterOrDigit(c)) { + hash = 31 * hash + c; + } + } + + hashes.add(handleNoAlphaNumeric(value, hash)); + } + + private static long handleNoAlphaNumeric(String value, long hash) { + if (hash == 0) { + for (char c : value.toCharArray()) { + hash = 31 * hash + toLower(c); + } + } + return hash; + } + + /** + * tokenize on everything except alpha-numeric characters + */ + private void addOriginalMatchHash(String value) { + long hash = 0; + for (char c : value.toCharArray()) { + c = toLower(c); + if (isLetterOrDigit(c)) { + hash = 31 * hash + c; + } else { + addHash(hash); + hash = 0; + } + } + addHash(hash); + } + + /** + * tokenize on whitespace and include only alpha-numeric characters in hash + */ + private void addPartialFuzzyPuncHash(String value) { + long hash = 0; + for (char c : value.toCharArray()) { + if (c == ' ' || c == '\n' || c == '\r' || c == '\t') { + addHash(hash); + hash = 0; + } else { + c = toLower(c); + if (isLetterOrDigit(c)) { + hash = 31 * hash + c; + } + } + } + addHash(hash); + } + + private void addHash(Long hash) { + if (hash != 0) { + hashes.add(hash); + } + } + + private static boolean isLetterOrDigit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'); + } + + private static char toLower(char c) { + if (c >= 'A' && c <= 'Z') { + return (char) (c + 32); + } + return c; + } + + public static void main(String[] args) { + SearchTermHash app = new SearchTermHash(); + System.out.println(app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.WHOLE_MATCH)); + System.out.println( + app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.TOKENIZE_WHITESPACE)); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java new file mode 100644 index 00000000000..1c74ec832fe --- /dev/null +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java @@ -0,0 +1,153 @@ +/******************************************************************************* + * Copyright (c) 2018 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.orcs.db.internal.search.handlers; + +import java.util.Iterator; +import java.util.List; +import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash; +import org.eclipse.osee.orcs.db.internal.search.SearchTermHash; +import org.eclipse.osee.orcs.db.internal.sql.AbstractSqlWriter; +import org.eclipse.osee.orcs.db.internal.sql.SqlHandler; +import org.eclipse.osee.orcs.db.internal.sql.TableEnum; + +/** + * @author Ryan D. Brooks + */ +public class AttributeHashQueryHandler extends SqlHandler<CriteriaAttributeHash> { + private String attTxsAlias; + private String attAlias; + private String artAlias; + private CriteriaAttributeHash criteria; + private final SearchTermHash hasher = new SearchTermHash(); + private List<List<Long>> hashes; + + @Override + public void setData(CriteriaAttributeHash criteria) { + this.criteria = criteria; + } + + @Override + public void addTables(AbstractSqlWriter writer) { + hashes = hasher.getTermHashes(criteria.getValues(), criteria.getOption()); + int hashTableCount = countNeededHashTables(); + + for (int i = 0; i < hashTableCount; i++) { + writer.addTable(TableEnum.SEARCH_HASH_TABLE); + } + attAlias = writer.addTable(TableEnum.ATTRIBUTE_TABLE); + artAlias = writer.getMainTableAlias(TableEnum.ARTIFACT_TABLE); + attTxsAlias = writer.addTable(TableEnum.TXS_TABLE); + } + + private int countNeededHashTables() { + int hashTableCount = 0; + Iterator<List<Long>> iterator = hashes.iterator(); + List<Long> singleTokens = iterator.next(); + if (!singleTokens.isEmpty()) { + hashTableCount++; + } + while (iterator.hasNext()) { + List<Long> list = iterator.next(); + hashTableCount += list.size(); + } + return hashTableCount; + } + + /** + * Combine all single tokens together using a simple in clause + */ + private void writeHashTableForSingleTokens(AbstractSqlWriter writer, String hshAlias, List<Long> singleTokens) { + writer.write("("); + writer.writeEqualsParameterAnd(hshAlias, "app_id", 1); + if (singleTokens.size() == 1) { + writer.writeEqualsParameter(hshAlias, "hash", singleTokens.get(0)); + } else { + writer.write(hshAlias); + writer.write(".hash IN ("); + boolean notFirst = false; + for (Long hash : singleTokens) { + if (notFirst) { + writer.write(", ?"); + } else { + writer.write("?"); + notFirst = true; + } + writer.addParameter(hash); + } + writer.write(")"); + } + writer.write(" AND "); + writer.writeEquals(hshAlias, attAlias, "gamma_id"); + writer.write(")"); + } + + private void addHashPredicates(AbstractSqlWriter writer) { + Iterator<String> aliases = writer.getAliases(TableEnum.SEARCH_HASH_TABLE).iterator(); + Iterator<List<Long>> hashIterator = hashes.iterator(); + String prevHshAlias = aliases.next(); + List<Long> singleTokens = hashIterator.next(); + + writer.write("("); + boolean hasSingleTokens = !singleTokens.isEmpty(); + if (hasSingleTokens) { + writeHashTableForSingleTokens(writer, prevHshAlias, singleTokens); + } + + boolean useOr = hasSingleTokens; + while (hashIterator.hasNext()) { + if (useOr) { + writer.write(" OR\n"); + } else { + useOr = true; + } + writer.write(" ("); + + boolean joinGammaIds = false; + for (Long hash : hashIterator.next()) { + String hshAlias = aliases.next(); + if (joinGammaIds) { + writer.writeEquals(prevHshAlias, hshAlias, "gamma_id"); + writer.write(" AND "); + } else { + joinGammaIds = true; + } + writer.write(hshAlias); + writer.write(".app_id = ? AND "); + writer.addParameter(1); + writer.write(hshAlias); + writer.write(".hash = ? AND "); + writer.addParameter(hash); + prevHshAlias = hshAlias; + } + writer.writeEquals(prevHshAlias, attAlias, "gamma_id"); + writer.write(")"); + } + writer.write(") AND\n "); + } + + @Override + public void addPredicates(AbstractSqlWriter writer) { + addHashPredicates(writer); + if (criteria.getAttributeType().isValid()) { + writer.writeEqualsParameterAnd(attAlias, "attr_type_id", criteria.getAttributeType()); + } + writer.writeEquals(attAlias, artAlias, "art_id"); + writer.write(" AND "); + writer.writeEquals(attAlias, attTxsAlias, "gamma_id"); + writer.write(" AND "); + writer.write(writer.getTxBranchFilter(attTxsAlias)); + } + + @Override + public int getPriority() { + return SqlHandlerPriority.ATTRIBUTE_TOKENIZED_VALUE.ordinal(); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java index 11fb853958c..706e4187484 100644 --- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java @@ -20,6 +20,7 @@ import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactGuids; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAssociatedArtId; +import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists; @@ -93,6 +94,7 @@ public final class SqlHandlerFactoryUtil { } private static void addArtifactHandlers(Map<Class<? extends Criteria>, Class<? extends SqlHandler<?>>> handleMap) { + handleMap.put(CriteriaAttributeHash.class, AttributeHashQueryHandler.class); handleMap.put(CriteriaArtifactGuids.class, ArtifactGuidSqlHandler.class); handleMap.put(CriteriaArtifactIds.class, ArtifactIdsSqlHandler.class); handleMap.put(CriteriaArtifactType.class, ArtifactTypeSqlHandler.class); diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java index 4d8e0baf1b7..44881656ccc 100644 --- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java @@ -17,6 +17,7 @@ import java.util.Set; import java.util.concurrent.Future; import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; import org.eclipse.osee.framework.jdk.core.type.OseeCoreException; import org.eclipse.osee.framework.jdk.core.util.Collections; @@ -28,6 +29,7 @@ import org.eclipse.osee.orcs.OrcsSession; import org.eclipse.osee.orcs.core.ds.IndexerData; import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer; import org.eclipse.osee.orcs.data.AttributeTypes; +import org.eclipse.osee.orcs.db.internal.search.SearchTermHash; import org.eclipse.osee.orcs.db.internal.search.indexer.callable.DeleteTagSetDatabaseTxCallable; import org.eclipse.osee.orcs.db.internal.search.indexer.callable.IndexerDatabaseStatisticsCallable; import org.eclipse.osee.orcs.db.internal.search.indexer.callable.PurgeAllTagsDatabaseCallable; @@ -155,4 +157,13 @@ public class QueryEngineIndexerImpl implements QueryEngineIndexer { systemCollector.removeCollector(collector); } -} + @Override + public int createTermHashes(Iterable<GammaId> gammaIds) { + return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(gammaIds); + } + + @Override + public int createTermHashes(AttributeTypeId attributeType) { + return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(attributeType); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java index 88534a8a0c2..a7ae5d1635a 100644 --- a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java +++ b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java @@ -10,15 +10,19 @@ *******************************************************************************/ package org.eclipse.osee.orcs.rest.model; +import java.util.List; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.eclipse.osee.framework.core.data.AttributeTypeId; +import org.eclipse.osee.framework.core.data.GammaId; /** * @author Roberto E. Escobar @@ -36,6 +40,13 @@ public interface IndexerEndpoint { @Consumes(MediaType.APPLICATION_JSON) Response indexResources(IndexResources options); + @POST + @Path("attributeType/{attributeType}") + int createTermHashes(@PathParam("attributeType") AttributeTypeId attributeType); + + @POST + int createTermHashes(List<GammaId> gammaIds); + @DELETE @Path("queue") Response deleteIndexQueue(); diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java index d42dea127c4..fa024331276 100644 --- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java +++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java @@ -25,6 +25,7 @@ import org.eclipse.osee.framework.core.data.BranchId; import org.eclipse.osee.framework.core.data.TransactionId; import org.eclipse.osee.framework.core.data.TransactionToken; import org.eclipse.osee.framework.core.data.UserId; +import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.jdk.core.type.MatchLocation; import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.orcs.OrcsApi; @@ -148,7 +149,7 @@ public class ArtifactEndpointImpl implements ArtifactEndpoint { } if (attributeType.isValid()) { if (exists) { - query.andAttributeIs(attributeType, value); + query.andAttribute(attributeType, value, QueryOption.WHOLE_MATCH, false); } else { query.andNotExists(attributeType, value); } diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java index 5e4a960233d..96eebcb30db 100644 --- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java +++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java @@ -19,8 +19,10 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; import org.eclipse.osee.framework.core.data.BranchId; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.framework.jdk.core.util.Collections; import org.eclipse.osee.jaxrs.OseeWebApplicationException; @@ -101,4 +103,13 @@ public class IndexerEndpointImpl implements IndexerEndpoint { return asResponse(modified); } -} + @Override + public int createTermHashes(AttributeTypeId attributeType) { + return getIndexer().createTermHashes(attributeType); + } + + @Override + public int createTermHashes(List<GammaId> gammaIds) { + return getIndexer().createTermHashes(gammaIds); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java index d8a1e084afb..dc7321c705a 100644 --- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java +++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java @@ -25,10 +25,9 @@ import org.eclipse.osee.framework.core.data.IRelationType; import org.eclipse.osee.framework.core.data.RelationTypeId; import org.eclipse.osee.framework.core.data.RelationTypeSide; import org.eclipse.osee.framework.core.data.TransactionId; -import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.core.enums.RelationSide; -import org.eclipse.osee.framework.core.data.RelationTypeId; import org.eclipse.osee.framework.core.data.ValueKind; +import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.core.enums.RelationSide; import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.orcs.data.ArtifactReadable; @@ -122,7 +121,7 @@ public interface QueryBuilder extends Query { <R> ArtifactQuerySelection<R> selectInto(R receiver); public <T> void selectAtt(AttributeTypeToken<T> attributeType, Consumer<T> consumer); - public static AttributeTypeToken ANY_ATTRIBUTE_TYPE = + public static AttributeTypeToken<Object> ANY_ATTRIBUTE_TYPE = AttributeTypeToken.valueOf(Long.MIN_VALUE, "Any Attribute Type"); QueryBuilder includeDeletedArtifacts(); @@ -317,6 +316,12 @@ public interface QueryBuilder extends Query { QueryBuilder followNoSelect(RelationTypeSide relationTypeSide, ArtifactTypeToken artifacType); + QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive); + + QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive); + + QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive); + /** * @deprecated use follow instead, currently still needed only for ORCS script */ diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java index dc857765714..bc7b81fe220 100644 --- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java +++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java @@ -14,7 +14,9 @@ import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.Future; +import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; /** @@ -28,7 +30,7 @@ public interface QueryIndexer { Callable<List<Future<?>>> indexResources(Iterable<Long> gammaIds, IndexerCollector... collector); - void indexAttrTypeIds(Iterable<Long> gammaIds); + void indexAttrTypeIds(Iterable<Long> attrTypeIds); void indexMissingByAttrTypeIds(Iterable<Long> attrTypeIds); @@ -38,4 +40,7 @@ public interface QueryIndexer { CancellableCallable<Integer> purgeAllIndexes(); -} + int createTermHashes(Iterable<GammaId> gammaIds); + + int createTermHashes(AttributeTypeId attributeType); +}
\ No newline at end of file |