diff options
author | Ryan D. Brooks | 2018-02-27 14:15:51 +0000 |
---|---|---|
committer | Ryan D. Brooks | 2020-12-15 22:27:47 +0000 |
commit | e401af2380dbd5bfae80a53fa789ada18d7a352d (patch) | |
tree | b964e6c9eda1c3587408f81ac10f911a45c07b61 | |
parent | 9f81f171bd2e8c493d46e063d980a0f1905b6558 (diff) | |
download | org.eclipse.osee-e401af2380dbd5bfae80a53fa789ada18d7a352d.tar.gz org.eclipse.osee-e401af2380dbd5bfae80a53fa789ada18d7a352d.tar.xz org.eclipse.osee-e401af2380dbd5bfae80a53fa789ada18d7a352d.zip |
feature: Create faster query indexer
- Provides a faster and more flexible approach for indexing the
- attribute values for various kinds of search including whole match,
- tokenized by whitespace, and tokenized by non-alphanumeric. Any one
- of the proceeding options can be combined with case sensitivity.
Change-Id: I97077898e9827786c806460afbb98301c1d5ebbd
16 files changed, 546 insertions, 17 deletions
diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java index 84ef46dde10..8456cd6f596 100644 --- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java +++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/QueryOption.java @@ -17,6 +17,7 @@ package org.eclipse.osee.framework.core.enums; * @author John Misinco */ public enum QueryOption { + NONE, CASE__MATCH, CASE__IGNORE, @@ -30,7 +31,12 @@ public enum QueryOption { // matching the token order TOKEN_MATCH_ORDER__ANY, - TOKEN_MATCH_ORDER__MATCH; + TOKEN_MATCH_ORDER__MATCH, + + // can be used to match one value or any values (i.e. using OR) but not all values (i.e. using AND) + WHOLE_MATCH, + TOKENIZE_WHITESPACE, + TOKENIZE_NON_ALPHANUMERIC; public static QueryOption getTokenOrderType(boolean matchOrder) { return matchOrder ? TOKEN_MATCH_ORDER__MATCH : TOKEN_MATCH_ORDER__ANY; diff --git a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java index fb08444ed1c..1eaa6c21e74 100644 --- a/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java +++ b/plugins/org.eclipse.osee.framework.core/src/org/eclipse/osee/framework/core/enums/SqlTable.java @@ -680,6 +680,14 @@ public class SqlTable extends NamedBase { OSEE_OAUTH_TOKEN_TABLE.createIndex("OSEE_OAUTH_TOKEN__TK_IDX", false, OSEE_OAUTH_TOKEN_TOKEN_KEY.getName()); } + public static final SqlTable SEARCH_HASH = new SqlTable("osee_search_hash", "sh", 1); + public static final SqlColumn SEARCH_HASH_APP_ID = TXS_TABLE.addColumn("APP_ID", JDBCType.BIGINT); + public static final SqlColumn SEARCH_HASH_HASH = TXS_TABLE.addColumn("HASH", JDBCType.BIGINT); + public static final SqlColumn SEARCH_HASH_GAMMA_ID = TXS_TABLE.addColumn("GAMMA_ID", JDBCType.BIGINT); + static { + TXS_TABLE.setPrimaryKeyConstraint(SEARCH_HASH_APP_ID, SEARCH_HASH_HASH, SEARCH_HASH_GAMMA_ID); + } + private final String aliasPrefix; private final ObjectType objectType; private final ChainingArrayList<@NonNull SqlColumn> columns; diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java index ddc35d233cc..53a84ac99c9 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryData.java @@ -54,6 +54,7 @@ import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactGuids; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactTxComment; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType; +import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists; @@ -287,8 +288,10 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch { return branch; } - public void select(AttributeTypeToken attributeType) { + @Override + public QueryBuilder select(AttributeTypeToken attributeType) { this.attributeType = attributeType; + return this; } @Override @@ -501,6 +504,21 @@ public final class QueryData implements QueryBuilder, HasOptions, HasBranch { } @Override + public QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) { + return addAndCheck(new CriteriaAttributeHash(attributeType, value, option, caseSensitive)); + } + + @Override + public QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) { + return addAndCheck(new CriteriaAttributeHash(attributeType, values, option, caseSensitive)); + } + + @Override + public QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive) { + return addAndCheck(new CriteriaAttributeHash(value, option, caseSensitive)); + } + + @Override public QueryBuilder andAttributeIs(AttributeTypeId attributeType, String... values) { return andAttributeIs(attributeType, Arrays.asList(values)); } diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java index 1d36b335ab1..6d77fcdbe45 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/QueryEngineIndexer.java @@ -18,6 +18,7 @@ import java.util.Set; import java.util.concurrent.Future; import org.eclipse.osee.framework.core.OrcsTokenService; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; import org.eclipse.osee.orcs.OrcsSession; import org.eclipse.osee.orcs.search.IndexerCollector; @@ -46,4 +47,8 @@ public interface QueryEngineIndexer { void indexAttrTypeIds(OrcsSession session, OrcsTokenService tokenService, Iterable<Long> attrTypeIds); void indexAttrTypeMissingOnly(OrcsTokenService tokenService, Iterable<Long> attrTypeIds); + + int createTermHashes(Iterable<GammaId> gammaIds); + + int createTermHashes(AttributeTypeId attributeType); }
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java new file mode 100644 index 00000000000..b960a673ff3 --- /dev/null +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/ds/criteria/CriteriaAttributeHash.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2018 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.orcs.core.ds.criteria; + +import java.util.Collection; +import java.util.Collections; +import org.eclipse.osee.framework.core.data.AttributeTypeId; +import org.eclipse.osee.framework.core.enums.QueryOption; +import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; +import org.eclipse.osee.orcs.core.ds.Criteria; +import org.eclipse.osee.orcs.core.ds.Options; + +/** + * @author Ryan D. Brooks + */ +public class CriteriaAttributeHash extends Criteria { + private final AttributeTypeId attributeType; + private final Collection<String> values; + private final QueryOption option; + private final boolean caseSensitive; + + public CriteriaAttributeHash(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive) { + this.attributeType = attributeType; + this.values = values; + this.option = option; + this.caseSensitive = caseSensitive; + } + + public CriteriaAttributeHash(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive) { + this(attributeType, Collections.singletonList(value), option, caseSensitive); + } + + public CriteriaAttributeHash(String value, QueryOption option, boolean caseSensitive) { + this(AttributeTypeId.SENTINEL, value, option, caseSensitive); + } + + public AttributeTypeId getAttributeType() { + return attributeType; + } + + public Collection<String> getValues() { + return values; + } + + public QueryOption getOption() { + return option; + } + + public boolean isCaseSensitive() { + return caseSensitive; + } + + @Override + public void checkValid(Options options) { + if (values.isEmpty()) { + throw new OseeArgumentException("CriteriaAttributeHash requires at least one value to match"); + } + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java index f7750b3fe5c..2c7f02cbbf6 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/indexer/QueryIndexerImpl.java @@ -20,6 +20,7 @@ import java.util.concurrent.Future; import org.eclipse.osee.framework.core.OrcsTokenService; import org.eclipse.osee.framework.core.data.AttributeTypeGeneric; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; import org.eclipse.osee.orcs.OrcsSession; import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer; @@ -90,4 +91,13 @@ public class QueryIndexerImpl implements QueryIndexer { return engineIndexer.purgeAllIndexes(session); } -} + @Override + public int createTermHashes(Iterable<GammaId> gammaIds) { + return engineIndexer.createTermHashes(gammaIds); + } + + @Override + public int createTermHashes(AttributeTypeId attributeType) { + return engineIndexer.createTermHashes(attributeType); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java index 0a4fbbcfaa7..f5374a87a42 100644 --- a/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java +++ b/plugins/org.eclipse.osee.orcs.core/src/org/eclipse/osee/orcs/core/internal/transaction/TransactionBuilderImpl.java @@ -42,12 +42,12 @@ import org.eclipse.osee.framework.core.enums.CoreArtifactTokens; import org.eclipse.osee.framework.core.enums.CoreArtifactTypes; import org.eclipse.osee.framework.core.enums.CoreAttributeTypes; import org.eclipse.osee.framework.core.enums.CoreTupleTypes; +import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.core.enums.RelationSorter; import org.eclipse.osee.framework.core.enums.SqlTable; import org.eclipse.osee.framework.jdk.core.type.Id; import org.eclipse.osee.framework.jdk.core.type.NamedId; import org.eclipse.osee.framework.jdk.core.type.OseeCoreException; -import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.framework.jdk.core.util.Conditions; import org.eclipse.osee.orcs.KeyValueOps; import org.eclipse.osee.orcs.OrcsApi; @@ -172,12 +172,12 @@ public class TransactionBuilderImpl implements TransactionBuilder { @Override public List<ArtifactToken> createArtifacts(ArtifactTypeToken artifactType, ArtifactId parent, List<String> names) { - ResultSet<ArtifactReadable> results = - queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).and(CoreAttributeTypes.Name, - names).getResults(); - if (!results.isEmpty()) { - throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", results.size(), - artifactType, results.getList()); + List<ArtifactToken> duplicates = + queryFactory.fromBranch(getBranch()).andTypeEquals(artifactType).andAttribute(CoreAttributeTypes.Name, names, + QueryOption.TOKENIZE_WHITESPACE, false).loadArtifactTokens(); + if (!duplicates.isEmpty()) { + throw new OseeCoreException("Found %s artifacts of type %s with duplicate names: %s", duplicates.size(), + artifactType, duplicates); } List<ArtifactToken> tokens = new ArrayList<>(names.size()); diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java new file mode 100644 index 00000000000..337daab8f0a --- /dev/null +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/SearchTermHash.java @@ -0,0 +1,216 @@ +/******************************************************************************* + * Copyright (c) 218 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.orcs.db.internal.search; + +import static org.eclipse.osee.jdbc.JdbcConstants.JDBC__MAX_FETCH_SIZE; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import org.eclipse.osee.framework.core.data.AttributeTypeId; +import org.eclipse.osee.framework.core.data.GammaId; +import org.eclipse.osee.framework.core.enums.QueryOption; +import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; +import org.eclipse.osee.jdbc.JdbcClient; +import org.eclipse.osee.jdbc.JdbcStatement; +import org.eclipse.osee.jdbc.OseePreparedStatement; +import org.eclipse.osee.orcs.db.internal.sql.join.IdJoinQuery; +import org.eclipse.osee.orcs.db.internal.sql.join.SqlJoinFactory; + +/** + * Instances of this class should not be shared among threads. Use multiple term normalization rules and then store the + * unique set of resulting hashes. Each normalization starts with making all letters lower-case.</br> + * <li>whole match = no tokenize; keep only alpha-numeric characters + * <li>original = tokenize on characters that are not digits or letters + * <li>partial with fuzzy punc = tokenize on whitespace; strip all punctuation + * <li>maybe not = tokenize on whitespace; strip leading and trailing punctuation; replace remaining punctuation with + * spaces + * <ul> + * <li>{Requirement_Name} + * <li>use new tablespace called osee_search Replace TagProcessor and use XmlTextInputStream + * + * @author Ryan D. Brooks + */ +public class SearchTermHash { + private static final String SELECT_MISSING_ATTRIBUTES = + "SELECT gamma_id, value FROM osee_attribute att WHERE attr_type_id = ? AND value is not null AND not exists (SELECT 1 FROM osee_search_hash has WHERE has.gamma_id = att.gamma_id)"; + + private static final String SELECT_ATTRIBUTES_BY_TYPE = + "SELECT gamma_id, value FROM osee_attribute WHERE attr_type_id = ? AND value is not null"; + + private static final String SELECT_ATTRIBUTES_BY_GAMMA = + "SELECT att.gamma_id, value FROM osee_attribute att, osee_join_id WHERE query_id = ? AND id = gamma_id AND value is not null"; + + private final SqlJoinFactory joinFactory; + private final JdbcClient jdbcClient; + private OseePreparedStatement insertStatement; + private final HashSet<Long> hashes = new HashSet<>(); + private final List<List<Long>> resultHashes = new ArrayList<>(); + private final List<Long> singleTokens = new ArrayList<>(); + + public SearchTermHash() { + this(null, null); + } + + public SearchTermHash(JdbcClient jdbcClient, SqlJoinFactory joinFactory) { + this.jdbcClient = jdbcClient; + this.joinFactory = joinFactory; + } + + public int createTermHashes(AttributeTypeId attributeType) { + return createTermHashes(SELECT_ATTRIBUTES_BY_TYPE, attributeType); + } + + public int createTermHashes(Iterable<GammaId> gammaIds) { + try (IdJoinQuery joinQuery = joinFactory.createIdJoinQuery()) { + joinQuery.addAndStore(gammaIds); + return createTermHashes(SELECT_ATTRIBUTES_BY_GAMMA, joinQuery.getQueryId()); + } + } + + private int createTermHashes(String sql, Object... data) { + insertStatement = + jdbcClient.getBatchStatement("INSERT INTO osee_search_hash (app_id, hash, gamma_id) VALUES (?, ?, ?)"); + int loaded = jdbcClient.runQuery(this::createTermHashes, JDBC__MAX_FETCH_SIZE, sql, data); + insertStatement.execute(); + return loaded; + } + + public List<List<Long>> getTermHashes(Collection<String> values, QueryOption queryOption) { + resultHashes.clear(); + singleTokens.clear(); + resultHashes.add(singleTokens); + for (String value : values) { + switch (queryOption) { + case TOKENIZE_NON_ALPHANUMERIC: + addOriginalMatchHash(value); + break; + case TOKENIZE_WHITESPACE: + addPartialFuzzyPuncHash(value); + break; + case WHOLE_MATCH: + getWholeMatchHash(value); + break; + default: + throw new OseeArgumentException("Unexpected query option %s", queryOption); + } + copyHashes(); + } + return resultHashes; + } + + private void copyHashes() { + if (hashes.size() == 1) { + singleTokens.add(hashes.iterator().next()); + } else { + resultHashes.add(new ArrayList<>(hashes)); + } + hashes.clear(); + } + + private void createTermHashes(JdbcStatement stmt) { + String value = stmt.getString("value"); + addOriginalMatchHash(value); + addPartialFuzzyPuncHash(value); + getWholeMatchHash(value); + + for (Long hash : hashes) { + insertStatement.addToBatch(1, hash, stmt.getLong("gamma_id")); + } + hashes.clear(); + } + + /** + * don't tokenize and include only alpha-numeric characters in hash. If the value contains only non alpha-numeric + * characters, then the hash is computed using all of its characters. + */ + public void getWholeMatchHash(String value) { + long hash = 0; + for (char c : value.toCharArray()) { + c = toLower(c); + if (isLetterOrDigit(c)) { + hash = 31 * hash + c; + } + } + + hashes.add(handleNoAlphaNumeric(value, hash)); + } + + private static long handleNoAlphaNumeric(String value, long hash) { + if (hash == 0) { + for (char c : value.toCharArray()) { + hash = 31 * hash + toLower(c); + } + } + return hash; + } + + /** + * tokenize on everything except alpha-numeric characters + */ + private void addOriginalMatchHash(String value) { + long hash = 0; + for (char c : value.toCharArray()) { + c = toLower(c); + if (isLetterOrDigit(c)) { + hash = 31 * hash + c; + } else { + addHash(hash); + hash = 0; + } + } + addHash(hash); + } + + /** + * tokenize on whitespace and include only alpha-numeric characters in hash + */ + private void addPartialFuzzyPuncHash(String value) { + long hash = 0; + for (char c : value.toCharArray()) { + if (c == ' ' || c == '\n' || c == '\r' || c == '\t') { + addHash(hash); + hash = 0; + } else { + c = toLower(c); + if (isLetterOrDigit(c)) { + hash = 31 * hash + c; + } + } + } + addHash(hash); + } + + private void addHash(Long hash) { + if (hash != 0) { + hashes.add(hash); + } + } + + private static boolean isLetterOrDigit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'); + } + + private static char toLower(char c) { + if (c >= 'A' && c <= 'Z') { + return (char) (c + 32); + } + return c; + } + + public static void main(String[] args) { + SearchTermHash app = new SearchTermHash(); + System.out.println(app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.WHOLE_MATCH)); + System.out.println( + app.getTermHashes(Arrays.asList("{Artifact_Name}", "Artifact Name"), QueryOption.TOKENIZE_WHITESPACE)); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java new file mode 100644 index 00000000000..1c74ec832fe --- /dev/null +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/AttributeHashQueryHandler.java @@ -0,0 +1,153 @@ +/******************************************************************************* + * Copyright (c) 2018 Boeing. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Boeing - initial API and implementation + *******************************************************************************/ +package org.eclipse.osee.orcs.db.internal.search.handlers; + +import java.util.Iterator; +import java.util.List; +import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash; +import org.eclipse.osee.orcs.db.internal.search.SearchTermHash; +import org.eclipse.osee.orcs.db.internal.sql.AbstractSqlWriter; +import org.eclipse.osee.orcs.db.internal.sql.SqlHandler; +import org.eclipse.osee.orcs.db.internal.sql.TableEnum; + +/** + * @author Ryan D. Brooks + */ +public class AttributeHashQueryHandler extends SqlHandler<CriteriaAttributeHash> { + private String attTxsAlias; + private String attAlias; + private String artAlias; + private CriteriaAttributeHash criteria; + private final SearchTermHash hasher = new SearchTermHash(); + private List<List<Long>> hashes; + + @Override + public void setData(CriteriaAttributeHash criteria) { + this.criteria = criteria; + } + + @Override + public void addTables(AbstractSqlWriter writer) { + hashes = hasher.getTermHashes(criteria.getValues(), criteria.getOption()); + int hashTableCount = countNeededHashTables(); + + for (int i = 0; i < hashTableCount; i++) { + writer.addTable(TableEnum.SEARCH_HASH_TABLE); + } + attAlias = writer.addTable(TableEnum.ATTRIBUTE_TABLE); + artAlias = writer.getMainTableAlias(TableEnum.ARTIFACT_TABLE); + attTxsAlias = writer.addTable(TableEnum.TXS_TABLE); + } + + private int countNeededHashTables() { + int hashTableCount = 0; + Iterator<List<Long>> iterator = hashes.iterator(); + List<Long> singleTokens = iterator.next(); + if (!singleTokens.isEmpty()) { + hashTableCount++; + } + while (iterator.hasNext()) { + List<Long> list = iterator.next(); + hashTableCount += list.size(); + } + return hashTableCount; + } + + /** + * Combine all single tokens together using a simple in clause + */ + private void writeHashTableForSingleTokens(AbstractSqlWriter writer, String hshAlias, List<Long> singleTokens) { + writer.write("("); + writer.writeEqualsParameterAnd(hshAlias, "app_id", 1); + if (singleTokens.size() == 1) { + writer.writeEqualsParameter(hshAlias, "hash", singleTokens.get(0)); + } else { + writer.write(hshAlias); + writer.write(".hash IN ("); + boolean notFirst = false; + for (Long hash : singleTokens) { + if (notFirst) { + writer.write(", ?"); + } else { + writer.write("?"); + notFirst = true; + } + writer.addParameter(hash); + } + writer.write(")"); + } + writer.write(" AND "); + writer.writeEquals(hshAlias, attAlias, "gamma_id"); + writer.write(")"); + } + + private void addHashPredicates(AbstractSqlWriter writer) { + Iterator<String> aliases = writer.getAliases(TableEnum.SEARCH_HASH_TABLE).iterator(); + Iterator<List<Long>> hashIterator = hashes.iterator(); + String prevHshAlias = aliases.next(); + List<Long> singleTokens = hashIterator.next(); + + writer.write("("); + boolean hasSingleTokens = !singleTokens.isEmpty(); + if (hasSingleTokens) { + writeHashTableForSingleTokens(writer, prevHshAlias, singleTokens); + } + + boolean useOr = hasSingleTokens; + while (hashIterator.hasNext()) { + if (useOr) { + writer.write(" OR\n"); + } else { + useOr = true; + } + writer.write(" ("); + + boolean joinGammaIds = false; + for (Long hash : hashIterator.next()) { + String hshAlias = aliases.next(); + if (joinGammaIds) { + writer.writeEquals(prevHshAlias, hshAlias, "gamma_id"); + writer.write(" AND "); + } else { + joinGammaIds = true; + } + writer.write(hshAlias); + writer.write(".app_id = ? AND "); + writer.addParameter(1); + writer.write(hshAlias); + writer.write(".hash = ? AND "); + writer.addParameter(hash); + prevHshAlias = hshAlias; + } + writer.writeEquals(prevHshAlias, attAlias, "gamma_id"); + writer.write(")"); + } + writer.write(") AND\n "); + } + + @Override + public void addPredicates(AbstractSqlWriter writer) { + addHashPredicates(writer); + if (criteria.getAttributeType().isValid()) { + writer.writeEqualsParameterAnd(attAlias, "attr_type_id", criteria.getAttributeType()); + } + writer.writeEquals(attAlias, artAlias, "art_id"); + writer.write(" AND "); + writer.writeEquals(attAlias, attTxsAlias, "gamma_id"); + writer.write(" AND "); + writer.write(writer.getTxBranchFilter(attTxsAlias)); + } + + @Override + public int getPriority() { + return SqlHandlerPriority.ATTRIBUTE_TOKENIZED_VALUE.ordinal(); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java index 3304a1b6bcb..917438e58f8 100644 --- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/handlers/SqlHandlerFactoryUtil.java @@ -24,6 +24,7 @@ import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactIds; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactTxComment; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaArtifactType; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAssociatedArtId; +import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeHash; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeKeywords; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeRaw; import org.eclipse.osee.orcs.core.ds.criteria.CriteriaAttributeTypeExists; @@ -96,6 +97,7 @@ public final class SqlHandlerFactoryUtil { } private static void addArtifactHandlers(Map<Class<? extends Criteria>, Class<? extends SqlHandler<?>>> handleMap) { + handleMap.put(CriteriaAttributeHash.class, AttributeHashQueryHandler.class); handleMap.put(CriteriaArtifactGuids.class, ArtifactGuidSqlHandler.class); handleMap.put(CriteriaArtifactIds.class, ArtifactIdsSqlHandler.class); handleMap.put(CriteriaArtifactType.class, ArtifactTypeSqlHandler.class); diff --git a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java index cd9e510a4f2..e5ad4d3d8c4 100644 --- a/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java +++ b/plugins/org.eclipse.osee.orcs.db/src/org/eclipse/osee/orcs/db/internal/search/indexer/QueryEngineIndexerImpl.java @@ -19,6 +19,7 @@ import java.util.Set; import java.util.concurrent.Future; import org.eclipse.osee.framework.core.OrcsTokenService; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; import org.eclipse.osee.framework.jdk.core.type.OseeCoreException; import org.eclipse.osee.framework.jdk.core.util.Collections; @@ -29,6 +30,7 @@ import org.eclipse.osee.logger.Log; import org.eclipse.osee.orcs.OrcsSession; import org.eclipse.osee.orcs.core.ds.IndexerData; import org.eclipse.osee.orcs.core.ds.QueryEngineIndexer; +import org.eclipse.osee.orcs.db.internal.search.SearchTermHash; import org.eclipse.osee.orcs.db.internal.search.indexer.callable.DeleteTagSetDatabaseTxCallable; import org.eclipse.osee.orcs.db.internal.search.indexer.callable.IndexerDatabaseStatisticsCallable; import org.eclipse.osee.orcs.db.internal.search.indexer.callable.PurgeAllTagsDatabaseCallable; @@ -157,4 +159,13 @@ public class QueryEngineIndexerImpl implements QueryEngineIndexer { systemCollector.removeCollector(collector); } -} + @Override + public int createTermHashes(Iterable<GammaId> gammaIds) { + return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(gammaIds); + } + + @Override + public int createTermHashes(AttributeTypeId attributeType) { + return new SearchTermHash(jdbcClient, joinFactory).createTermHashes(attributeType); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java index a797002d14f..a5ec89fe6ff 100644 --- a/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java +++ b/plugins/org.eclipse.osee.orcs.rest.model/src/org/eclipse/osee/orcs/rest/model/IndexerEndpoint.java @@ -13,15 +13,19 @@ package org.eclipse.osee.orcs.rest.model; +import java.util.List; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.eclipse.osee.framework.core.data.AttributeTypeId; +import org.eclipse.osee.framework.core.data.GammaId; /** * @author Roberto E. Escobar @@ -39,6 +43,13 @@ public interface IndexerEndpoint { @Consumes(MediaType.APPLICATION_JSON) Response indexResources(IndexResources options); + @POST + @Path("attributeType/{attributeType}") + int createTermHashes(@PathParam("attributeType") AttributeTypeId attributeType); + + @POST + int createTermHashes(List<GammaId> gammaIds); + @DELETE @Path("queue") Response deleteIndexQueue(); diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java index e941503b071..ac17a31ff1c 100644 --- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java +++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/ArtifactEndpointImpl.java @@ -30,6 +30,7 @@ import org.eclipse.osee.framework.core.data.BranchId; import org.eclipse.osee.framework.core.data.TransactionId; import org.eclipse.osee.framework.core.data.TransactionToken; import org.eclipse.osee.framework.core.data.UserId; +import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.jdk.core.type.MatchLocation; import org.eclipse.osee.framework.jdk.core.type.MultipleItemsExist; import org.eclipse.osee.framework.jdk.core.type.OseeArgumentException; diff --git a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java index bf81aacbae9..671134dcfa0 100644 --- a/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java +++ b/plugins/org.eclipse.osee.orcs.rest/src/org/eclipse/osee/orcs/rest/internal/IndexerEndpointImpl.java @@ -22,8 +22,10 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; import org.eclipse.osee.framework.core.data.BranchId; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.framework.jdk.core.util.Collections; import org.eclipse.osee.jaxrs.OseeWebApplicationException; @@ -104,4 +106,13 @@ public class IndexerEndpointImpl implements IndexerEndpoint { return asResponse(modified); } -} + @Override + public int createTermHashes(AttributeTypeId attributeType) { + return getIndexer().createTermHashes(attributeType); + } + + @Override + public int createTermHashes(List<GammaId> gammaIds) { + return getIndexer().createTermHashes(gammaIds); + } +}
\ No newline at end of file diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java index 8267e07650a..71b185b0d5f 100644 --- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java +++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryBuilder.java @@ -27,9 +27,8 @@ import org.eclipse.osee.framework.core.data.AttributeTypeToken; import org.eclipse.osee.framework.core.data.RelationTypeSide; import org.eclipse.osee.framework.core.data.RelationTypeToken; import org.eclipse.osee.framework.core.data.TransactionId; -import org.eclipse.osee.framework.core.enums.QueryOption; -import org.eclipse.osee.framework.core.data.RelationTypeId; import org.eclipse.osee.framework.core.data.ValueKind; +import org.eclipse.osee.framework.core.enums.QueryOption; import org.eclipse.osee.framework.core.enums.RelationSide; import org.eclipse.osee.framework.jdk.core.type.ResultSet; import org.eclipse.osee.orcs.data.ArtifactReadable; @@ -123,7 +122,7 @@ public interface QueryBuilder extends Query { <R> ArtifactQuerySelection<R> selectInto(R receiver); public <T> void selectAtt(AttributeTypeToken<T> attributeType, Consumer<T> consumer); - public static AttributeTypeToken ANY_ATTRIBUTE_TYPE = + public static AttributeTypeToken<Object> ANY_ATTRIBUTE_TYPE = AttributeTypeToken.valueOf(Long.MIN_VALUE, "Any Attribute Type"); QueryBuilder includeDeletedArtifacts(); @@ -324,6 +323,12 @@ public interface QueryBuilder extends Query { QueryBuilder followNoSelect(RelationTypeSide relationTypeSide, ArtifactTypeToken artifacType); + QueryBuilder andAttribute(AttributeTypeId attributeType, String value, QueryOption option, boolean caseSensitive); + + QueryBuilder andAttribute(AttributeTypeId attributeType, Collection<String> values, QueryOption option, boolean caseSensitive); + + QueryBuilder andAttribute(String value, QueryOption option, boolean caseSensitive); + /** * @deprecated use follow instead, currently still needed only for ORCS script */ diff --git a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java index 1f6ee433d13..4dbcebc8c28 100644 --- a/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java +++ b/plugins/org.eclipse.osee.orcs/src/org/eclipse/osee/orcs/search/QueryIndexer.java @@ -18,7 +18,9 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.Future; import org.eclipse.osee.framework.core.data.AttributeTypeGeneric; +import org.eclipse.osee.framework.core.data.AttributeTypeId; import org.eclipse.osee.framework.core.data.Branch; +import org.eclipse.osee.framework.core.data.GammaId; import org.eclipse.osee.framework.core.executor.CancellableCallable; /** @@ -32,7 +34,7 @@ public interface QueryIndexer { Callable<List<Future<?>>> indexResources(Iterable<Long> gammaIds, IndexerCollector... collector); - void indexAttrTypeIds(Iterable<Long> gammaIds); + void indexAttrTypeIds(Iterable<Long> attrTypeIds); void indexMissingByAttrTypeIds(Iterable<Long> attrTypeIds); @@ -42,4 +44,7 @@ public interface QueryIndexer { CancellableCallable<Integer> purgeAllIndexes(); -} + int createTermHashes(Iterable<GammaId> gammaIds); + + int createTermHashes(AttributeTypeId attributeType); +}
\ No newline at end of file |