From 3e544c125bad79fd25dd9b741b5616ada998abb3 Mon Sep 17 00:00:00 2001 From: "Brian M. Long" Date: Thu, 31 Oct 2024 14:55:42 -0400 Subject: [PATCH 1/5] initial community-module (incomplete/breaking) --- community-module/.gitignore | 12 + community-module/README.md | 1 + community-module/pom.xml | 86 +++++ community-module/rad.ps1 | 74 ++++ community-module/rad.sh | 71 ++++ .../asie/provider/ShardRegistryProvider.java | 28 ++ .../asie/service/ShardStateService.java | 80 ++++ .../asie/service/SolrShardRegistry.java | 363 ++++++++++++++++++ .../alfresco-global.properties | 28 ++ .../log4j2.properties | 3 + .../module-context.xml | 18 + .../module.properties | 10 + pom.xml | 1 + 13 files changed, 775 insertions(+) create mode 100644 community-module/.gitignore create mode 100644 community-module/README.md create mode 100644 community-module/pom.xml create mode 100644 community-module/rad.ps1 create mode 100644 community-module/rad.sh create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java create mode 100644 community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties create mode 100644 community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties create mode 100644 community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml create mode 100644 community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties diff --git a/community-module/.gitignore b/community-module/.gitignore new file mode 100644 index 0000000..e59065e --- /dev/null +++ b/community-module/.gitignore @@ -0,0 +1,12 @@ +# Maven +target +pom.xml.versionsBackup + +# Eclipse +.project +.classpath +.settings +.vscode + +# IDEA +/.idea/ diff --git a/community-module/README.md b/community-module/README.md new file mode 100644 index 0000000..1aa10d1 --- /dev/null +++ b/community-module/README.md @@ -0,0 +1 @@ +# ASIE Platform Module Library diff --git a/community-module/pom.xml b/community-module/pom.xml new file mode 100644 index 0000000..25fdbab --- /dev/null +++ b/community-module/pom.xml @@ -0,0 +1,86 @@ + + 4.0.0 + + + com.inteligr8.alfresco + asie-platform-module-parent + 1.0-SNAPSHOT + ../ + + + asie-community-platform-module + jar + + ASIE Platform Module for ACS Community + + + 5.2.0 + 23.3.0 + + + + + + org.alfresco + acs-community-packaging + ${alfresco.platform.version} + pom + import + + + + + + + com.inteligr8.alfresco + asie-shared + ${project.version} + provided + + + + + org.alfresco + alfresco-repository + provided + + + + + junit + junit + test + + + org.mockito + mockito-core + test + + + + + + + io.repaint.maven + tiles-maven-plugin + 2.40 + true + + + + com.inteligr8.ootbee:beedk-acs-platform-module-tile:[1.1.6,2.0.0) + + + + + + + + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + + diff --git a/community-module/rad.ps1 b/community-module/rad.ps1 new file mode 100644 index 0000000..61bcb2f --- /dev/null +++ b/community-module/rad.ps1 @@ -0,0 +1,74 @@ + +function discoverArtifactId { + $script:ARTIFACT_ID=(mvn -q -Dexpression=project"."artifactId -DforceStdout help:evaluate) +} + +function rebuild { + echo "Rebuilding project ..." + mvn process-classes +} + +function start_ { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad process-classes +} + +function start_log { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad "-Ddocker.showLogs" process-classes +} + +function stop_ { + discoverArtifactId + echo "Stopping Docker containers that supported rapid application development ..." + docker container ls --filter name=${ARTIFACT_ID}-* + echo "Stopping containers ..." + docker container stop (docker container ls -q --filter name=${ARTIFACT_ID}-*) + echo "Removing containers ..." + docker container rm (docker container ls -aq --filter name=${ARTIFACT_ID}-*) +} + +function tail_logs { + param ( + $container + ) + + discoverArtifactId + docker container logs -f (docker container ls -q --filter name=${ARTIFACT_ID}-${container}) +} + +function list { + discoverArtifactId + docker container ls --filter name=${ARTIFACT_ID}-* +} + +switch ($args[0]) { + "start" { + start_ + } + "start_log" { + start_log + } + "stop" { + stop_ + } + "restart" { + stop_ + start_ + } + "rebuild" { + rebuild + } + "tail" { + tail_logs $args[1] + } + "containers" { + list + } + default { + echo "Usage: .\rad.ps1 [ start | start_log | stop | restart | rebuild | tail {container} | containers ]" + } +} + +echo "Completed!" + diff --git a/community-module/rad.sh b/community-module/rad.sh new file mode 100644 index 0000000..7cb0a80 --- /dev/null +++ b/community-module/rad.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +discoverArtifactId() { + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate` +} + +rebuild() { + echo "Rebuilding project ..." + mvn process-classes +} + +start() { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad process-classes +} + +start_log() { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad -Ddocker.showLogs process-classes +} + +stop() { + discoverArtifactId + echo "Stopping Docker containers that supported rapid application development ..." + docker container ls --filter name=${ARTIFACT_ID}-* + echo "Stopping containers ..." + docker container stop `docker container ls -q --filter name=${ARTIFACT_ID}-*` + echo "Removing containers ..." + docker container rm `docker container ls -aq --filter name=${ARTIFACT_ID}-*` +} + +tail_logs() { + discoverArtifactId + docker container logs -f `docker container ls -q --filter name=${ARTIFACT_ID}-$1` +} + +list() { + discoverArtifactId + docker container ls --filter name=${ARTIFACT_ID}-* +} + +case "$1" in + start) + start + ;; + start_log) + start_log + ;; + stop) + stop + ;; + restart) + stop + start + ;; + rebuild) + rebuild + ;; + tail) + tail_logs $2 + ;; + containers) + list + ;; + *) + echo "Usage: ./rad.sh [ start | start_log | stop | restart | rebuild | tail {container} | containers ]" + exit 1 +esac + +echo "Completed!" + diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java new file mode 100644 index 0000000..5355d0f --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java @@ -0,0 +1,28 @@ +package com.inteligr8.alfresco.asie.provider; + +import org.alfresco.repo.index.shard.ShardRegistry; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; + +import com.inteligr8.alfresco.asie.Constants; + +@Configuration +public class ShardRegistryProvider extends AbstractProvider { + + /** + * This allows for the selection of the primary or first ShardRegistry + * registered in the Spring BeanFactory. + * + * @return A ShardRegistry. + */ + @Bean(Constants.BEAN_SHARD_REGISTRY) + @Qualifier(Constants.QUALIFIER_ASIE) + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public ShardRegistry selectBean() { + return this.getPrimary(ShardRegistry.class); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java new file mode 100644 index 0000000..c93c5ef --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java @@ -0,0 +1,80 @@ +package com.inteligr8.alfresco.asie.service; + +import java.io.Serializable; +import java.util.Arrays; + +import org.alfresco.repo.cache.SimpleCache; +import org.alfresco.repo.index.shard.ShardInstance; +import org.alfresco.repo.index.shard.ShardState; +import org.alfresco.service.cmr.attributes.AttributeService; +import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.Constants; +import com.inteligr8.alfresco.asie.enterprise.EnterpriseConstants; + +@Component +public class ShardStateService implements com.inteligr8.alfresco.asie.spi.ShardStateService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier(Constants.QUALIFIER_ASIE) + private AttributeService attrService; + + @Autowired + @Qualifier(Constants.BEAN_SHARD_STATE_CACHE) + private SimpleCache shardStateCache; + + public void clear() { + this.logger.info("Removing all nodes/shards from the shard registry"); + + // this clears the state from the backend database + this.attrService.removeAttributes(EnterpriseConstants.ATTR_SHARD_STATE); + this.attrService.removeAttributes(EnterpriseConstants.ATTR_SHARD_SUBSCRIPTION); + this.attrService.removeAttributes(Constants.ATTR_ASIE); + + // this clears the state from Hazelcast + this.shardStateCache.clear(); + this.shardToGuidCache.clear(); + } + + public void remove(Serializable... keys) { + if (keys.length == 0) + throw new IllegalArgumentException(); + + this.logger.info("Removing from the shard registry: {}", Arrays.toString(keys)); + + Serializable[] shardStateKeys = keys; + Serializable[] shardSubKeys; + if (EnterpriseConstants.ATTR_SHARD_STATE.equals(keys[0])) { + shardSubKeys = ArrayUtils.clone(keys); + shardSubKeys[0] = EnterpriseConstants.ATTR_SHARD_SUBSCRIPTION; + } else { + shardStateKeys = ArrayUtils.addFirst(keys, EnterpriseConstants.ATTR_SHARD_STATE); + shardSubKeys = ArrayUtils.addFirst(keys, EnterpriseConstants.ATTR_SHARD_SUBSCRIPTION); + } + + ShardState shardState = (ShardState) this.attrService.getAttribute(shardStateKeys); + + // this clears the state from the backend database + this.attrService.removeAttribute(shardStateKeys); + this.attrService.removeAttribute(shardSubKeys); + + // this clears the state from Hazelcast + if (shardState != null) { + this.shardStateCache.remove(shardState.getShardInstance()); + this.shardToGuidCache.remove(shardState.getShardInstance()); + } + } + + public void iterate(AttributeQueryCallback callback) { + this.attrService.getAttributes(callback, EnterpriseConstants.ATTR_SHARD_STATE); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java new file mode 100644 index 0000000..9a96726 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java @@ -0,0 +1,363 @@ +package com.inteligr8.alfresco.asie.service; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Map.Entry; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.alfresco.repo.cache.SimpleCache; +import org.alfresco.repo.index.shard.Floc; +import org.alfresco.repo.index.shard.Shard; +import org.alfresco.repo.index.shard.ShardInstance; +import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.repo.index.shard.ShardRegistry; +import org.alfresco.repo.index.shard.ShardState; +import org.alfresco.repo.search.impl.QueryParserUtils; +import org.alfresco.repo.search.impl.parsers.AlfrescoFunctionEvaluationContext; +import org.alfresco.repo.search.impl.parsers.CMISLexer; +import org.alfresco.repo.search.impl.parsers.FTSLexer; +import org.alfresco.repo.search.impl.parsers.FTSParser; +import org.alfresco.repo.search.impl.parsers.FTSQueryParser; +import org.alfresco.repo.search.impl.querymodel.Conjunction; +import org.alfresco.repo.search.impl.querymodel.Constraint; +import org.alfresco.repo.search.impl.querymodel.Disjunction; +import org.alfresco.repo.search.impl.querymodel.FunctionalConstraint; +import org.alfresco.repo.search.impl.querymodel.QueryEngine; +import org.alfresco.repo.search.impl.querymodel.QueryModelFactory; +import org.alfresco.repo.search.impl.querymodel.QueryOptions; +import org.alfresco.repo.search.impl.querymodel.QueryOptions.Connective; +import org.alfresco.repo.search.impl.querymodel.impl.BaseConstraint; +import org.alfresco.repo.search.impl.querymodel.impl.lucene.LuceneQueryBuilderComponent; +import org.alfresco.service.cmr.attributes.AttributeService; +import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.search.SearchParameters; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.tree.CommonTree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEvent; +import org.springframework.extensions.surf.util.AbstractLifecycleBean; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.Constants; +import com.inteligr8.alfresco.asie.model.Node; +import com.inteligr8.alfresco.asie.model.ShardSet; + +@Component +public class SolrShardRegistry extends AbstractLifecycleBean implements ShardRegistry { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final Pattern coreShardPattern = Pattern.compile("(.+)-[0-9]+"); + + @Autowired + private ShardStateService sss; + + @Autowired + @Qualifier(Constants.QUALIFIER_ASIE) + private AttributeService attrService; + + @Autowired + private NamespaceService namespaceService; + + @Autowired + @Qualifier(Constants.BEAN_SHARD_STATE_CACHE) + private SimpleCache onlineShardCache; + + @Autowired + @Qualifier(Constants.BEAN_OFFILINE_SHARD_STATE_CACHE) + private SimpleCache offlineShardCache; + + @Autowired + @Qualifier(Constants.BEAN_CORE_EXPLICIT_CACHE) + private SimpleCache coreExplicitIdCache; + + @Autowired + @Qualifier(Constants.BEAN_FLOC_CACHE) + private SimpleCache flocCache; + + @Value("${inteligr8.asie.registerUnknownShardOffline}") + private boolean registerOffline; + + @Value("${inteligr8.asie.offlineIdleShardInSeconds}") + private int offlineIdleShardInSeconds; + + @Value("${inteligr8.asie.forgetOfflineShardInSeconds}") + private int forgetOfflineShardInSeconds; + + @Override + protected void onBootstrap(ApplicationEvent event) { + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + switch ((String) keys[2]) { + case Constants.ATTR_STATE: + ShardState shardNodeState = (ShardState) value; + ShardInstance shardNode = shardNodeState.getShardInstance(); + cacheShard(shardNode, shardNodeState, (String) keys[1]); + + if (ShardMethodEnum.EXPLICIT_ID.toString().equals(shardNodeState.getPropertyBag().get("shard.method"))) { + String coreName = shardNode.getShard().getFloc().getPropertyBag().get("coreName"); + if (coreName != null && !coreExplicitIdCache.contains(coreName)) { + String property = shardNodeState.getPropertyBag().get("shard.key"); + QName propertyQname = QName.createQName(property, namespaceService); + + logger.debug("Mapping core to explicit ID: {} => {}", coreName, propertyQname); + coreExplicitIdCache.put(coreName, propertyQname); + } + } + + return true; + default: + return true; + + } + } + }, Constants.ATTR_ASIE_NODES); + } + + @Override + protected void onShutdown(ApplicationEvent event) { + } + + protected void cacheShard(ShardInstance shardNode, ShardState shardNodeState, String nodeId) { + SimpleCache shardCache = this.onlineShardCache; + ShardState cachedShardNodeState = this.onlineShardCache.get(shardNode); + if (cachedShardNodeState == null) { + cachedShardNodeState = this.offlineShardCache.get(shardNode); + shardCache = this.offlineShardCache; + } + + if (cachedShardNodeState == null) { + Boolean online = (Boolean) this.attrService.getAttribute(Constants.ATTR_ASIE_NODES, nodeId, Constants.ATTR_ONLINE); + if (online != null) { + if (online.booleanValue()) { + this.onlineShardCache.put(shardNode, cachedShardNodeState); + } else { + this.offlineShardCache.put(shardNode, cachedShardNodeState); + } + } else { + if (this.registerOffline) { + this.offlineShardCache.put(shardNode, cachedShardNodeState); + } else { + this.onlineShardCache.put(shardNode, cachedShardNodeState); + } + } + } else if (cachedShardNodeState.getLastIndexedTxId() < shardNodeState.getLastIndexedTxId()) { + shardCache.put(shardNode, shardNodeState); + } + } + + protected void fixFlocPropertyBag(ShardState shardNodeState) { + Floc floc = shardNodeState.getShardInstance().getShard().getFloc(); + if (floc.getPropertyBag().isEmpty()) { + for (Entry prop : shardNodeState.getPropertyBag().entrySet()) { + if (prop.getKey().startsWith("shard.")) { + floc.getPropertyBag().put(prop.getKey(), prop.getValue()); + } else if (prop.getKey().equals("coreName")) { + String coreName = this.extractCoreName(prop.getValue()); + if (coreName != null) + floc.getPropertyBag().put(prop.getKey(), coreName); + } + } + } + } + + protected String extractCoreName(String coreShardName) { + Matcher matcher = coreShardPattern.matcher(coreShardName); + if (!matcher.matches()) + return null; + return matcher.group(1); + } + + @Override + public void registerShardState(ShardState shardNodeState) { + ShardInstance shardNode = shardNodeState.getShardInstance(); + Node node = new Node(shardNode); + this.fixFlocPropertyBag(shardNodeState); + this.cacheShard(shardNode, shardNodeState, node.getId()); + } + + @Override + public Map>> getFlocs() { + Map>> flocs = new HashMap<>(); + + for (ShardInstance shardNode : this.onlineShardCache.getKeys()) { + Floc floc = shardNode.getShard().getFloc(); + + Map> shards = flocs.get(floc); + if (shards == null) + flocs.put(floc, shards = new HashMap<>()); + + Set shardNodeStates = shards.get(shardNode.getShard()); + if (shardNodeStates == null) + shards.put(shardNode.getShard(), shardNodeStates = new HashSet<>()); + + ShardState shardNodeState = this.onlineShardCache.get(shardNode); + if (shardNodeState != null) // in case it was removed during the looping (very rare) + shardNodeStates.add(shardNodeState); + } + + return flocs; + } + + @Override + public void purge() { + this.sss.clear(); + } + + @Override + public void purgeAgedOutShards() { + long onlineExpired = System.currentTimeMillis() - this.offlineIdleShardInSeconds * 1000L; + long offlineExpired = System.currentTimeMillis() - this.forgetOfflineShardInSeconds * 1000L; + + for (ShardInstance shardNode : this.onlineShardCache.getKeys()) { + ShardState shardNodeState = this.onlineShardCache.get(shardNode); + if (shardNodeState.getLastUpdated() < onlineExpired) { + this.logger.warn("Taking shard offline: {}", shardNode); + this.onlineShardCache.remove(shardNode); + this.offlineShardCache.put(shardNode, shardNodeState); + } + } + + for (ShardInstance shardNode : this.offlineShardCache.getKeys()) { + ShardState shardNodeState = this.offlineShardCache.get(shardNode); + if (shardNodeState.getLastUpdated() < offlineExpired) { + this.logger.info("Forgetting about already offline shard: {}", shardNode); + this.offlineShardCache.remove(shardNode); + } + } + } + + @Override + public QName getExplicitIdProperty(String coreName) { + return this.coreExplicitIdCache.get(coreName); + } + + @Override + public Set getShardInstanceList(String coreName) { + Set shardIds = new HashSet<>(); + + for (ShardInstance shardNode : this.onlineShardCache.getKeys()) { + shardIds.add(shardNode.getShard().getInstance()); + } + + return shardIds; + } + + @Override + public OptionalInt getShardInstanceByTransactionTimestamp(String coreId, long txnTimestamp) { + throw new UnsupportedOperationException(); + } + + @Override + public List getIndexSlice(SearchParameters searchParameters) { + for (Floc floc : this.flocCache.getKeys()) { + Set shardIds = new HashSet<>(); + + switch (floc.getShardMethod()) { + case EXPLICIT_ID: + String property = floc.getPropertyBag().get("shard.key"); + // check filters and other parameters + if (searchParameters.getQuery() != null) { + SearchTerm term = this.extractPropertySearchTeam(searchParameters, property); + if (term != null && term.operator.equals("=")) { + try { + shardIds.add(Integer.parseInt(term.value)); + } catch (NumberFormatException nfe) { + // skip + } + } + } + break; + } + } + searchParameters.get + // TODO Auto-generated method stub + return null; + } + + private SearchTerm extractPropertySearchTeam(SearchParameters searchParameters, String property) { + switch (searchParameters.getLanguage()) { + case SearchService.LANGUAGE_CMIS_ALFRESCO: + case SearchService.LANGUAGE_CMIS_STRICT: + case SearchService.LANGUAGE_INDEX_CMIS: + case SearchService.LANGUAGE_SOLR_CMIS: + return this.extractCmisPropertySearchTerm(searchParameters, property, "="); + case SearchService.LANGUAGE_FTS_ALFRESCO: + case SearchService.LANGUAGE_INDEX_ALFRESCO: + case SearchService.LANGUAGE_INDEX_FTS_ALFRESCO: + case SearchService.LANGUAGE_LUCENE: + case SearchService.LANGUAGE_SOLR_ALFRESCO: + case SearchService.LANGUAGE_SOLR_FTS_ALFRESCO: + return this.extractFtsPropertySearchTerm(searchParameters, "=@" + property); + default: + return null; + } + } + + @Autowired + private QueryEngine queryEngine; + + @Autowired + private DictionaryService dictionaryService; + + private SearchTerm extractFtsPropertySearchTerm(SearchParameters searchParameters, String field) { + // TODO include filter and other possible constraints + + if (searchParameters.getQuery() == null) + return null; + + CharStream cs = new ANTLRStringStream(searchParameters.getQuery()); + FTSLexer lexer = new FTSLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + FTSParser parser = new FTSParser(tokens); + parser.setDefaultFieldConjunction(searchParameters.getDefaultFTSOperator().equals(Operator.AND)); + parser.setMode(searchParameters.getDefaultFTSOperator().equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); + CommonTree ftsNode = (CommonTree) parser.ftsQuery().getTree(); + } + + private SearchTerm extractCmisPropertySearchTerm(SearchParameters searchParameters, String field, String operator) { + // TODO include filter and other possible constraints + + if (searchParameters.getQuery() == null) + return null; + + CharStream cs = new ANTLRStringStream(searchParameters.getQuery()); + CMISLexer lexer = new CMISLexer(); + CommonTokenStream tokens = new CommonTokenStream(lexer); + FTSParser parser = new FTSParser(tokens); + parser.setDefaultFieldConjunction(searchParameters.getDefaultFTSOperator().equals(Operator.AND)); + parser.setMode(searchParameters.getDefaultFTSOperator().equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); + CommonTree ftsNode = (CommonTree) parser.ftsQuery().getTree(); + } + + + + private class SearchTerm { + + private String field; + private String operator; + private String value; + + } + +} diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties new file mode 100644 index 0000000..ef89240 --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties @@ -0,0 +1,28 @@ + +inteligr8.asie.registerUnknownShardOffline=false +inteligr8.asie.idleShardExpirationInSeconds=${} + + + +# maxItems needs to be greater than total shards, including HA instances +cache.offlineShardStateSharedCache.tx.maxItems=1024 +cache.offlineShardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.offlineShardStateSharedCache.maxItems=1024 +cache.offlineShardStateSharedCache.timeToLiveSeconds=1800 +cache.offlineShardStateSharedCache.maxIdleSeconds=0 +cache.offlineShardStateSharedCache.cluster.type=fully-distributed +cache.offlineShardStateSharedCache.backup-count=1 +cache.offlineShardStateSharedCache.eviction-policy=LRU +cache.offlineShardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy +cache.offlineShardStateSharedCache.readBackupData=false + +cache.coreExplicitIdSharedCache.tx.maxItems=1024 +cache.coreExplicitIdSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.coreExplicitIdSharedCache.maxItems=1024 +cache.coreExplicitIdSharedCache.timeToLiveSeconds=1800 +cache.coreExplicitIdSharedCache.maxIdleSeconds=0 +cache.coreExplicitIdSharedCache.cluster.type=fully-distributed +cache.coreExplicitIdSharedCache.backup-count=1 +cache.coreExplicitIdSharedCache.eviction-policy=LRU +cache.coreExplicitIdSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy +cache.coreExplicitIdSharedCache.readBackupData=false diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties new file mode 100644 index 0000000..6c345f1 --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties @@ -0,0 +1,3 @@ + +logger.inteligr8-asie.name=com.inteligr8.alfresco.asie +logger.inteligr8-asie.level=INFO diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml new file mode 100644 index 0000000..0eebd98 --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties new file mode 100644 index 0000000..2d638d2 --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties @@ -0,0 +1,10 @@ +module.id=com_inteligr8_alfresco_${project.artifactId} +module.aliases= +module.title=${project.name} +module.description=${project.description} +module.version=${module.version} + +module.repo.version.min=23.0 + +# this is creating all sorts of problems; probably because of the non-standard versioning +module.depends.com.inteligr8.alfresco.cxf-jaxrs-platform-module=* diff --git a/pom.xml b/pom.xml index 0fb6d55..6ccf9b6 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ asie-api shared enterprise-module + community-module From 01d2f5ce23f877a4cce100f86455f4584d0f34db Mon Sep 17 00:00:00 2001 From: "Brian M. Long" Date: Fri, 1 Nov 2024 08:35:38 -0400 Subject: [PATCH 2/5] fix v1.1.x pom --- community-module/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community-module/pom.xml b/community-module/pom.xml index 25fdbab..f22cdf4 100644 --- a/community-module/pom.xml +++ b/community-module/pom.xml @@ -6,7 +6,7 @@ com.inteligr8.alfresco asie-platform-module-parent - 1.0-SNAPSHOT + 1.1-SNAPSHOT ../ From 3ecbf006dd6945029f75f0df2e59f9429432d729 Mon Sep 17 00:00:00 2001 From: "Brian M. Long" Date: Wed, 6 Nov 2024 13:24:54 -0500 Subject: [PATCH 3/5] added query parsing (incomplete) --- community-module/pom.xml | 2 +- .../alfresco/asie/cache/MultiValueCache.java | 67 ++++ .../asie/compute/CmisQueryInspector.java | 47 +++ .../asie/compute/FtsQueryInspector.java | 290 ++++++++++++++ .../alfresco/asie/compute/QueryInspector.java | 74 ++++ .../asie/compute/QueryInspectorFactory.java | 31 ++ .../asie/service/SolrShardRegistry.java | 368 ++++++++++-------- .../alfresco-global.properties | 20 +- .../log4j2.properties | 3 - .../asie/QueryConstraintUnitTest.java | 146 +++++++ .../alfresco/asie/TreeSerializer.java | 44 +++ 11 files changed, 929 insertions(+), 163 deletions(-) create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java create mode 100644 community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java create mode 100644 community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java diff --git a/community-module/pom.xml b/community-module/pom.xml index f22cdf4..359c4f3 100644 --- a/community-module/pom.xml +++ b/community-module/pom.xml @@ -17,7 +17,7 @@ 5.2.0 - 23.3.0 + 7.0.0 diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java new file mode 100644 index 0000000..d6d820c --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java @@ -0,0 +1,67 @@ +package com.inteligr8.alfresco.asie.cache; + +import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; + +import org.alfresco.repo.cache.SimpleCache; + +public class MultiValueCache> implements SimpleCache { + + private SimpleCache cache; + private Class collectionType; + + public MultiValueCache(SimpleCache cache, Class collectionType) { + this.cache = cache; + this.collectionType = collectionType; + } + + @SuppressWarnings("unchecked") + public boolean add(K key, V value) { + C c = this.cache.get(key); + if (c != null) + return c.add(value); + + try { + Constructor constructor = this.collectionType.getConstructor(); + c = (C) constructor.newInstance(); + this.cache.put(key, c); + return c.add(value); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException | InstantiationException e) { + throw new UnsupportedOperationException(e.getMessage(), e); + } + } + + @Override + public void clear() { + this.cache.clear(); + } + + @Override + public C get(K key) { + return this.cache.get(key); + } + + @Override + public boolean contains(K key) { + C c = this.cache.get(key); + return c == null ? false : !c.isEmpty(); + } + + @Override + public Collection getKeys() { + return this.cache.getKeys(); + } + + @Override + public void put(K key, C value) { + this.cache.put(key, value); + } + + @Override + public void remove(K key) { + this.cache.remove(key); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java new file mode 100644 index 0000000..7b0486b --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java @@ -0,0 +1,47 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.util.Set; + +import org.alfresco.repo.search.impl.parsers.CMISLexer; +import org.alfresco.repo.search.impl.parsers.CMISParser; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; +import org.apache.commons.collections4.SetUtils; +import org.springframework.stereotype.Component; + +@Component +public class CmisQueryInspector implements QueryInspector { + + private Set supportedLanguages = SetUtils.unmodifiableSet( + SearchService.LANGUAGE_CMIS_ALFRESCO, + SearchService.LANGUAGE_CMIS_STRICT, + SearchService.LANGUAGE_INDEX_CMIS, + SearchService.LANGUAGE_SOLR_CMIS); + + @Override + public Set getSupportedLanguages() { + return this.supportedLanguages; + } + + @Override + public QueryValue findRequiredProperty(String cmisQuery, Operator defaultOperator, QName property) throws RecognitionException { + Tree tree = this.parseCmis(cmisQuery, defaultOperator); + } + + protected Tree parseCmis(String cmisQuery, Operator defaultOperator) throws RecognitionException { + CharStream cs = new ANTLRStringStream(cmisQuery); + CMISLexer lexer = new CMISLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + CMISParser parser = new CMISParser(tokens); + CommonTree tree = (CommonTree) parser.query().getTree(); + return tree; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java new file mode 100644 index 0000000..cb972ba --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java @@ -0,0 +1,290 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Period; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.alfresco.repo.search.impl.parsers.FTSLexer; +import org.alfresco.repo.search.impl.parsers.FTSParser; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.repository.AssociationRef; +import org.alfresco.service.cmr.repository.ChildAssociationRef; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; +import org.apache.commons.collections4.SetUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class FtsQueryInspector implements QueryInspector { + + private final Logger logger = LoggerFactory.getLogger(FtsQueryInspector.class); + + private final Set supportedLanguages = SetUtils.unmodifiableSet( + SearchService.LANGUAGE_FTS_ALFRESCO, + SearchService.LANGUAGE_INDEX_FTS_ALFRESCO, + SearchService.LANGUAGE_SOLR_FTS_ALFRESCO, + SearchService.LANGUAGE_LUCENE); + + @Autowired + private NamespaceService namespaceService; + + @Override + public Set getSupportedLanguages() { + return this.supportedLanguages; + } + + @Override + public List findRequiredPropertyValues(String ftsQuery, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException { + Tree tree = this.parseFts(ftsQuery, defaultOperator); + tree = this.bypassSingleTermDisjunctions(tree); + if (tree == null) + return null; + + Collection trees = this.extractRequiredTerms(tree); + this.logger.trace("Found {} required terms in query: {}", trees.size(), ftsQuery); + this.filterPropertyTerms(trees, property); + this.logger.trace("Found {} required terms for property {} in query: {}", trees.size(), property, ftsQuery); + this.filterOutFuzzyTerms(trees); + this.logger.trace("Found {} required definitive terms for property {} in query: {}", trees.size(), property, ftsQuery); + + List values = new ArrayList<>(trees.size()); + for (Tree t : trees) + values.add(this.extractValue(t, dataTypeDef)); + return values; + } + + protected Tree parseFts(String ftsQuery, Operator defaultOperator) throws RecognitionException { + CharStream cs = new ANTLRStringStream(ftsQuery); + FTSLexer lexer = new FTSLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + FTSParser parser = new FTSParser(tokens); + parser.setDefaultFieldConjunction(defaultOperator.equals(Operator.AND)); + parser.setMode(defaultOperator.equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); + CommonTree tree = (CommonTree) parser.ftsQuery().getTree(); + return tree; + } + + protected Tree bypassSingleTermDisjunctions(Tree tree) { + while ("DISJUNCTION".equals(tree.getText()) && tree.getChildCount() == 1) + tree = tree.getChild(0); + if ("DISJUNCTION".equals(tree.getText())) + return null; + return tree; + } + + protected Collection extractRequiredTerms(Tree tree) { + while ("DISJUNCTION".equals(tree.getText()) && tree.getChildCount() == 1) + tree = tree.getChild(0); + + List terms = new LinkedList<>(); + + switch (tree.getText()) { + case "DISJUNCTION": + break; + case "CONJUNCTION": + for (int c = 0; c < tree.getChildCount(); c++) { + Collection subtrees = this.extractRequiredTerms(tree.getChild(c)); + if (subtrees == null || subtrees.isEmpty()) + continue; + terms.addAll(subtrees); + } + break; + case "DEFAULT": + terms.add(tree); + break; + default: + this.logger.warn("Unexpected/unsupported tree: {}", tree.getText()); + } + + return terms; + } + + protected Collection filterPropertyTerms(Collection trees, QName property) { + if (trees.isEmpty()) + return trees; + + Set prefixes = new HashSet<>(this.namespaceService.getPrefixes(property.getNamespaceURI())); + if (prefixes.isEmpty()) { + this.logger.warn("Unexpected/unsupported namespace: {}", property.getNamespaceURI()); + trees.clear(); + return trees; + } + + Iterator i = trees.iterator(); + + while (i.hasNext()) { + Tree tree = i.next(); + + if ("DEFAULT".equals(tree.getText())) + tree = tree.getChild(0); + + int skip = -1; + switch (tree.getText()) { + case "TERM": + case "PHRASE": + case "EXACT_TERM": + case "EXACT_PHRASE": + skip = 1; // skip the value child + break; + case "RANGE": + skip = 4; // skip the inclusive, start, end, inclusive children + break; + default: + } + + if (skip >= 0) { + Tree fieldRef = tree.getChild(skip); + if (!"FIELD_REF".equals(fieldRef.getText())) { + this.logger.warn("Unexpected/unsupported tree: {}", tree.getText()); + } else if (!fieldRef.getChild(0).getText().equals(property.getLocalName())) { + this.logger.trace("Found but ignoring property: {}", fieldRef.getChild(0).getText()); + } else { + Tree prefix = fieldRef.getChild(1); + if (!"PREFIX".equals(prefix.getText())) { + this.logger.warn("Unexpected/unsupported tree: {}", tree.getText()); + } else if (!prefixes.contains(prefix.getChild(0).getText())) { + this.logger.trace("Found but ignoring property: {}:{}", prefix.getChild(0).getText(), property.getLocalName()); + } else { + // this will skip the remove() + continue; + } + } + } + + i.remove(); + } + + return trees; + } + + protected Collection filterOutFuzzyTerms(Collection trees) { + if (trees.isEmpty()) + return trees; + + Iterator i = trees.iterator(); + + while (i.hasNext()) { + Tree tree = i.next(); + + if ("DEFAULT".equals(tree.getText())) + tree = tree.getChild(0); + + switch (tree.getText()) { + case "EXACT_TERM": + case "EXACT_PHRASE": + case "RANGE": + break; + default: + i.remove(); + } + } + + return trees; + } + + protected QueryValue extractValue(Tree tree, DataTypeDefinition dataTypeDef) { + if ("DEFAULT".equals(tree.getText())) + tree = tree.getChild(0); + + switch (tree.getText()) { + case "RANGE": + return this.extractRangeValue(tree, dataTypeDef); + default: + } + + String value = this.unquote(tree.getChild(0).getText()); + + switch (dataTypeDef.getName().getLocalName()) { + case "boolean": + return new QuerySingleValue(Boolean.parseBoolean(value)); + case "double": + return new QuerySingleValue(Double.parseDouble(value)); + case "float": + return new QuerySingleValue(Float.parseFloat(value)); + case "int": + return new QuerySingleValue(Integer.parseInt(value)); + case "long": + return new QuerySingleValue(Long.parseLong(value)); + case "date": + return new QuerySingleValue(this.evaluateAsDate(value)); + case "datetime": + return new QuerySingleValue(this.evaluateAsDateTime(value)); + case "period": + return new QuerySingleValue(Period.parse(value)); + case "qname": + return new QuerySingleValue(QName.createQName(value, this.namespaceService)); + case "noderef": + return new QuerySingleValue(new NodeRef(value)); + case "childassocref": + return new QuerySingleValue(new ChildAssociationRef(value)); + case "assocref": + return new QuerySingleValue(new AssociationRef(value)); + case "locale": + return new QuerySingleValue(new Locale(value)); + default: + return new QuerySingleValue(value); + } + } + + protected QueryRangeValue extractRangeValue(Tree tree, DataTypeDefinition dataTypeDef) { + boolean includeStart = "INCLUSIVE".equals(tree.getChild(0).getText()); + String start = this.unquote(tree.getChild(1).getText()); + String end = this.unquote(tree.getChild(2).getText()); + boolean includeEnd = "INCLUSIVE".equals(tree.getChild(3).getText()); + + switch (dataTypeDef.getName().getLocalName()) { + case "double": + return new QueryRangeValue(includeStart, Double.parseDouble(start), includeEnd, Double.parseDouble(end)); + case "float": + return new QueryRangeValue(includeStart, Float.parseFloat(start), includeEnd, Float.parseFloat(end)); + case "int": + return new QueryRangeValue(includeStart, Integer.parseInt(start), includeEnd, Integer.parseInt(end)); + case "long": + return new QueryRangeValue(includeStart, Long.parseLong(start), includeEnd, Long.parseLong(end)); + case "date": + return new QueryRangeValue(includeStart, this.evaluateAsDate(start), includeEnd, this.evaluateAsDate(end)); + case "datetime": + return new QueryRangeValue(includeStart, this.evaluateAsDateTime(start), includeEnd, this.evaluateAsDateTime(end)); + default: + throw new UnsupportedOperationException("The data type does not make sense for range evaluation: " + dataTypeDef.getName()); + } + } + + protected LocalDate evaluateAsDate(String str) { + if ("now".equalsIgnoreCase(str)) return LocalDate.now(); + else return LocalDate.parse(str); + } + + protected LocalDateTime evaluateAsDateTime(String str) { + if ("now".equalsIgnoreCase(str)) return LocalDateTime.now(); + else return LocalDateTime.parse(str); + } + + protected String unquote(String str) { + if (str.length() < 2) return str; + else if (str.charAt(0) == '\'' && str.charAt(str.length()-1) == '\'') return str.substring(1, str.length()-1); + else if (str.charAt(0) == '\"' && str.charAt(str.length()-1) == '\"') return str.substring(1, str.length()-1); + else return str; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java new file mode 100644 index 0000000..6a1348d --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java @@ -0,0 +1,74 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.util.List; +import java.util.Set; + +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.RecognitionException; + +public interface QueryInspector { + + Set getSupportedLanguages(); + + List findRequiredPropertyValues(String query, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException; + + + + public interface QueryValue { + + } + + public class QuerySingleValue implements QueryValue { + + private T value; + + public QuerySingleValue(T value) { + this.value = value; + } + + public T getValue() { + return value; + } + + @Override + public String toString() { + return this.value.toString(); + } + + } + + public class QueryRangeValue implements QueryValue { + + private boolean includeStart; + private T start; + private boolean includeEnd; + private T end; + + public QueryRangeValue(boolean includeStart, T start, boolean includeEnd, T end) { + this.includeStart = includeStart; + this.start = start; + this.includeEnd = includeEnd; + this.end = end; + } + + public boolean isIncludeStart() { + return includeStart; + } + + public boolean isIncludeEnd() { + return includeEnd; + } + + public T getStart() { + return start; + } + + public T getEnd() { + return end; + } + + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java new file mode 100644 index 0000000..bad416d --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java @@ -0,0 +1,31 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.util.List; +import java.util.Map; + +import org.alfresco.service.cmr.search.SearchParameters; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class QueryInspectorFactory implements InitializingBean { + + @Autowired + private List inspectors; + + private Map languageInspectorMap; + + @Override + public void afterPropertiesSet() throws Exception { + for (QueryInspector inspector : this.inspectors) { + for (String language : inspector.getSupportedLanguages()) + this.languageInspectorMap.put(language, inspector); + } + } + + public QueryInspector selectQueryInspector(SearchParameters searchParams) { + return this.languageInspectorMap.get(searchParams.getLanguage()); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java index 9a96726..c982712 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java @@ -8,8 +8,8 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.OptionalInt; import java.util.Map.Entry; +import java.util.OptionalInt; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,34 +21,15 @@ import org.alfresco.repo.index.shard.ShardInstance; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.alfresco.repo.index.shard.ShardRegistry; import org.alfresco.repo.index.shard.ShardState; -import org.alfresco.repo.search.impl.QueryParserUtils; -import org.alfresco.repo.search.impl.parsers.AlfrescoFunctionEvaluationContext; -import org.alfresco.repo.search.impl.parsers.CMISLexer; -import org.alfresco.repo.search.impl.parsers.FTSLexer; -import org.alfresco.repo.search.impl.parsers.FTSParser; -import org.alfresco.repo.search.impl.parsers.FTSQueryParser; -import org.alfresco.repo.search.impl.querymodel.Conjunction; -import org.alfresco.repo.search.impl.querymodel.Constraint; -import org.alfresco.repo.search.impl.querymodel.Disjunction; -import org.alfresco.repo.search.impl.querymodel.FunctionalConstraint; -import org.alfresco.repo.search.impl.querymodel.QueryEngine; -import org.alfresco.repo.search.impl.querymodel.QueryModelFactory; -import org.alfresco.repo.search.impl.querymodel.QueryOptions; -import org.alfresco.repo.search.impl.querymodel.QueryOptions.Connective; -import org.alfresco.repo.search.impl.querymodel.impl.BaseConstraint; -import org.alfresco.repo.search.impl.querymodel.impl.lucene.LuceneQueryBuilderComponent; +import org.alfresco.repo.lock.JobLockService; import org.alfresco.service.cmr.attributes.AttributeService; import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.search.SearchParameters; -import org.alfresco.service.cmr.search.SearchParameters.Operator; -import org.alfresco.service.cmr.search.SearchService; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; -import org.antlr.runtime.ANTLRStringStream; -import org.antlr.runtime.CharStream; -import org.antlr.runtime.CommonTokenStream; -import org.antlr.runtime.tree.CommonTree; +import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -59,6 +40,12 @@ import org.springframework.extensions.surf.util.AbstractLifecycleBean; import org.springframework.stereotype.Component; import com.inteligr8.alfresco.asie.Constants; +import com.inteligr8.alfresco.asie.cache.MultiValueCache; +import com.inteligr8.alfresco.asie.compute.QueryInspector; +import com.inteligr8.alfresco.asie.compute.QueryInspector.QueryRangeValue; +import com.inteligr8.alfresco.asie.compute.QueryInspector.QuerySingleValue; +import com.inteligr8.alfresco.asie.compute.QueryInspector.QueryValue; +import com.inteligr8.alfresco.asie.compute.QueryInspectorFactory; import com.inteligr8.alfresco.asie.model.Node; import com.inteligr8.alfresco.asie.model.ShardSet; @@ -68,6 +55,8 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg private final Logger logger = LoggerFactory.getLogger(this.getClass()); private final Pattern coreShardPattern = Pattern.compile("(.+)-[0-9]+"); + private final QName shardLock = QName.createQName(Constants.NAMESPACE_ASIE, "shardLock"); + @Autowired private ShardStateService sss; @@ -79,21 +68,30 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg private NamespaceService namespaceService; @Autowired - @Qualifier(Constants.BEAN_SHARD_STATE_CACHE) - private SimpleCache onlineShardCache; + private DictionaryService dictionaryService; + + @Autowired + private QueryInspectorFactory queryInspectorFactory; + + @Autowired + private JobLockService jobLockService; + + @Autowired + @Qualifier(Constants.BEAN_FLOC_SHARD_NODE_CACHE) + private SimpleCache>> flocShardNodeCache; + + @Autowired + @Qualifier(Constants.BEAN_ONLINE_SHARD_STATE_CACHE) + private SimpleCache onlineNodeShardStateCache; @Autowired @Qualifier(Constants.BEAN_OFFILINE_SHARD_STATE_CACHE) - private SimpleCache offlineShardCache; + private SimpleCache offlineNodeShardStateCache; @Autowired @Qualifier(Constants.BEAN_CORE_EXPLICIT_CACHE) private SimpleCache coreExplicitIdCache; - @Autowired - @Qualifier(Constants.BEAN_FLOC_CACHE) - private SimpleCache flocCache; - @Value("${inteligr8.asie.registerUnknownShardOffline}") private boolean registerOffline; @@ -105,64 +103,99 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg @Override protected void onBootstrap(ApplicationEvent event) { - this.attrService.getAttributes(new AttributeQueryCallback() { - @Override - public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { - switch ((String) keys[2]) { - case Constants.ATTR_STATE: - ShardState shardNodeState = (ShardState) value; - ShardInstance shardNode = shardNodeState.getShardInstance(); - cacheShard(shardNode, shardNodeState, (String) keys[1]); - - if (ShardMethodEnum.EXPLICIT_ID.toString().equals(shardNodeState.getPropertyBag().get("shard.method"))) { - String coreName = shardNode.getShard().getFloc().getPropertyBag().get("coreName"); - if (coreName != null && !coreExplicitIdCache.contains(coreName)) { - String property = shardNodeState.getPropertyBag().get("shard.key"); - QName propertyQname = QName.createQName(property, namespaceService); - - logger.debug("Mapping core to explicit ID: {} => {}", coreName, propertyQname); - coreExplicitIdCache.put(coreName, propertyQname); - } - } - - return true; - default: - return true; - + String lock = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + switch ((String) keys[2]) { + case Constants.ATTR_STATE: + ShardState shardNodeState = (ShardState) value; + ShardInstance shardNode = shardNodeState.getShardInstance(); + cacheShard(shardNode, shardNodeState, (String) keys[1]); + return true; + default: + return true; + + } } - } - }, Constants.ATTR_ASIE_NODES); + }, Constants.ATTR_ASIE_NODE_SHARD); + } finally { + this.jobLockService.releaseLock(lock, this.shardLock); + } } @Override protected void onShutdown(ApplicationEvent event) { } - - protected void cacheShard(ShardInstance shardNode, ShardState shardNodeState, String nodeId) { - SimpleCache shardCache = this.onlineShardCache; - ShardState cachedShardNodeState = this.onlineShardCache.get(shardNode); + + /** + * This is private because it must be wrapped in a cluster-safe lock + */ + private void cacheShard(ShardInstance shardNode, ShardState shardNodeState, String nodeShardId) { + ShardInstance detachedShardNode = this.detach(shardNode); + + SimpleCache shardCache = this.onlineNodeShardStateCache; + ShardState cachedShardNodeState = this.onlineNodeShardStateCache.get(detachedShardNode); if (cachedShardNodeState == null) { - cachedShardNodeState = this.offlineShardCache.get(shardNode); - shardCache = this.offlineShardCache; + cachedShardNodeState = this.offlineNodeShardStateCache.get(detachedShardNode); + shardCache = this.offlineNodeShardStateCache; } + + Shard shard = shardNode.getShard(); + this.putPutAdd(this.flocShardNodeCache, shard.getFloc(), shard.getInstance(), detachedShardNode); if (cachedShardNodeState == null) { - Boolean online = (Boolean) this.attrService.getAttribute(Constants.ATTR_ASIE_NODES, nodeId, Constants.ATTR_ONLINE); + Boolean online = (Boolean) this.attrService.getAttribute(Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_ONLINE); if (online != null) { if (online.booleanValue()) { - this.onlineShardCache.put(shardNode, cachedShardNodeState); + this.onlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); } else { - this.offlineShardCache.put(shardNode, cachedShardNodeState); + this.offlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); } } else { if (this.registerOffline) { - this.offlineShardCache.put(shardNode, cachedShardNodeState); + this.offlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); } else { - this.onlineShardCache.put(shardNode, cachedShardNodeState); + this.onlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); } } } else if (cachedShardNodeState.getLastIndexedTxId() < shardNodeState.getLastIndexedTxId()) { - shardCache.put(shardNode, shardNodeState); + // update the cached state if the state's last indexes transaction is later + shardCache.put(shardNode, this.detach(shardNodeState)); + } + + switch (shardNode.getShard().getFloc().getShardMethod()) { + case EXPLICIT_ID: + cacheExplicitShard(shardNode, shardNodeState); + break; + default: + } + } + + private void cacheExplicitShard(ShardInstance shardNode, ShardState shardNodeState) { + String coreName = shardNode.getShard().getFloc().getPropertyBag().get("coreName"); + if (coreName != null && !this.coreExplicitIdCache.contains(coreName)) { + String property = shardNodeState.getPropertyBag().get("shard.key"); + QName propertyQName = QName.createQName(property, this.namespaceService); + + this.logger.debug("Mapping core to explicit ID: {} => {}", coreName, propertyQName); + this.coreExplicitIdCache.put(coreName, propertyQName); + } + } + + @Override + public void registerShardState(ShardState shardNodeState) { + ShardInstance shardNode = shardNodeState.getShardInstance(); + Node node = new Node(shardNode); + this.fixFlocPropertyBag(shardNodeState); + + String lock = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.cacheShard(shardNode, shardNodeState, node.getId()); + this.persistShards(); + } finally { + this.jobLockService.releaseLock(lock, this.shardLock); } } @@ -182,25 +215,50 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg } protected String extractCoreName(String coreShardName) { - Matcher matcher = coreShardPattern.matcher(coreShardName); + Matcher matcher = this.coreShardPattern.matcher(coreShardName); if (!matcher.matches()) return null; return matcher.group(1); } - @Override - public void registerShardState(ShardState shardNodeState) { - ShardInstance shardNode = shardNodeState.getShardInstance(); - Node node = new Node(shardNode); - this.fixFlocPropertyBag(shardNodeState); - this.cacheShard(shardNode, shardNodeState, node.getId()); + /** + * This is private because it must be wrapped in a cluster-safe lock + */ + private void persistShards() { + long onlineExpired = System.currentTimeMillis() - this.offlineIdleShardInSeconds * 1000L; + long offlineExpired = System.currentTimeMillis() - this.forgetOfflineShardInSeconds * 1000L; + + for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { + String nodeShardId = new Node(shardNode).getId() + ";" + shardNode.getShard().getInstance(); + ShardState shardNodeState = this.onlineNodeShardStateCache.get(shardNode); + if (shardNodeState.getLastUpdated() < onlineExpired) { + this.logger.warn("Taking shard offline: {}", shardNode); + this.onlineNodeShardStateCache.remove(shardNode); + this.offlineNodeShardStateCache.put(shardNode, shardNodeState); + } else { + this.attrService.setAttribute(shardNodeState, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_STATE); + this.attrService.setAttribute(Boolean.TRUE, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_ONLINE); + } + } + + for (ShardInstance shardNode : this.offlineNodeShardStateCache.getKeys()) { + String nodeShardId = new Node(shardNode).getId() + ";" + shardNode.getShard().getInstance(); + ShardState shardNodeState = this.offlineNodeShardStateCache.get(shardNode); + if (shardNodeState.getLastUpdated() < offlineExpired) { + this.logger.info("Forgetting about already offline shard: {}", shardNode); + this.offlineNodeShardStateCache.remove(shardNode); + } else { + this.attrService.setAttribute(shardNodeState, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_STATE); + this.attrService.setAttribute(Boolean.FALSE, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_ONLINE); + } + } } @Override public Map>> getFlocs() { Map>> flocs = new HashMap<>(); - for (ShardInstance shardNode : this.onlineShardCache.getKeys()) { + for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { Floc floc = shardNode.getShard().getFloc(); Map> shards = flocs.get(floc); @@ -211,7 +269,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg if (shardNodeStates == null) shards.put(shardNode.getShard(), shardNodeStates = new HashSet<>()); - ShardState shardNodeState = this.onlineShardCache.get(shardNode); + ShardState shardNodeState = this.onlineNodeShardStateCache.get(shardNode); if (shardNodeState != null) // in case it was removed during the looping (very rare) shardNodeStates.add(shardNodeState); } @@ -229,20 +287,20 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg long onlineExpired = System.currentTimeMillis() - this.offlineIdleShardInSeconds * 1000L; long offlineExpired = System.currentTimeMillis() - this.forgetOfflineShardInSeconds * 1000L; - for (ShardInstance shardNode : this.onlineShardCache.getKeys()) { - ShardState shardNodeState = this.onlineShardCache.get(shardNode); + for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { + ShardState shardNodeState = this.onlineNodeShardStateCache.get(shardNode); if (shardNodeState.getLastUpdated() < onlineExpired) { this.logger.warn("Taking shard offline: {}", shardNode); - this.onlineShardCache.remove(shardNode); - this.offlineShardCache.put(shardNode, shardNodeState); + this.onlineNodeShardStateCache.remove(shardNode); + this.offlineNodeShardStateCache.put(shardNode, shardNodeState); } } - for (ShardInstance shardNode : this.offlineShardCache.getKeys()) { - ShardState shardNodeState = this.offlineShardCache.get(shardNode); + for (ShardInstance shardNode : this.offlineNodeShardStateCache.getKeys()) { + ShardState shardNodeState = this.offlineNodeShardStateCache.get(shardNode); if (shardNodeState.getLastUpdated() < offlineExpired) { this.logger.info("Forgetting about already offline shard: {}", shardNode); - this.offlineShardCache.remove(shardNode); + this.offlineNodeShardStateCache.remove(shardNode); } } } @@ -256,7 +314,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg public Set getShardInstanceList(String coreName) { Set shardIds = new HashSet<>(); - for (ShardInstance shardNode : this.onlineShardCache.getKeys()) { + for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { shardIds.add(shardNode.getShard().getInstance()); } @@ -270,94 +328,94 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg @Override public List getIndexSlice(SearchParameters searchParameters) { - for (Floc floc : this.flocCache.getKeys()) { - Set shardIds = new HashSet<>(); + if (searchParameters.getQuery() == null) + return Collections.emptyList(); + + List bestShards = null; + + for (Floc floc : this.flocShardMultiCache.getKeys()) { + List shards = new LinkedList<>(); switch (floc.getShardMethod()) { case EXPLICIT_ID: String property = floc.getPropertyBag().get("shard.key"); - // check filters and other parameters - if (searchParameters.getQuery() != null) { - SearchTerm term = this.extractPropertySearchTeam(searchParameters, property); - if (term != null && term.operator.equals("=")) { - try { - shardIds.add(Integer.parseInt(term.value)); - } catch (NumberFormatException nfe) { - // skip - } + QName propertyQName = QName.createQName(property, this.namespaceService); + DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); + + QueryInspector inspector = this.queryInspectorFactory.selectQueryInspector(searchParameters); + if (inspector == null) + continue; + + Set shardIds = new HashSet<>(); + List values = inspector.findRequiredPropertyValues(searchParameters.getQuery(), searchParameters.getDefaultOperator(), propertyQName, dtdef); + for (QueryValue value : values) { + if (value instanceof QuerySingleValue) { + @SuppressWarnings("unchecked") + Number num = ((QuerySingleValue) value).getValue(); + shardIds.add(num.intValue()); + } else if (value instanceof QueryRangeValue) { + @SuppressWarnings("unchecked") + QueryRangeValue num = (QueryRangeValue) value; + int start = num.getStart().intValue(); + if (!num.isIncludeStart()) + start++; + int end = num.getStart().intValue(); + if (!num.isIncludeEnd()) + end--; + for (int shardId = start; shardId <= end; shardId++) + shardIds.add(shardId); } } + + // shardIds to shardInstances break; + default: + // make no determination } + + if (!shards.isEmpty() && (bestShards == null || shards.size() < bestShards.size())) + bestShards = shards; } - searchParameters.get - // TODO Auto-generated method stub - return null; + + return bestShards; } - private SearchTerm extractPropertySearchTeam(SearchParameters searchParameters, String property) { - switch (searchParameters.getLanguage()) { - case SearchService.LANGUAGE_CMIS_ALFRESCO: - case SearchService.LANGUAGE_CMIS_STRICT: - case SearchService.LANGUAGE_INDEX_CMIS: - case SearchService.LANGUAGE_SOLR_CMIS: - return this.extractCmisPropertySearchTerm(searchParameters, property, "="); - case SearchService.LANGUAGE_FTS_ALFRESCO: - case SearchService.LANGUAGE_INDEX_ALFRESCO: - case SearchService.LANGUAGE_INDEX_FTS_ALFRESCO: - case SearchService.LANGUAGE_LUCENE: - case SearchService.LANGUAGE_SOLR_ALFRESCO: - case SearchService.LANGUAGE_SOLR_FTS_ALFRESCO: - return this.extractFtsPropertySearchTerm(searchParameters, "=@" + property); - default: - return null; - } + protected List getIndexSlice() { + } - @Autowired - private QueryEngine queryEngine; - - @Autowired - private DictionaryService dictionaryService; - - private SearchTerm extractFtsPropertySearchTerm(SearchParameters searchParameters, String field) { - // TODO include filter and other possible constraints - - if (searchParameters.getQuery() == null) - return null; - - CharStream cs = new ANTLRStringStream(searchParameters.getQuery()); - FTSLexer lexer = new FTSLexer(cs); - CommonTokenStream tokens = new CommonTokenStream(lexer); - FTSParser parser = new FTSParser(tokens); - parser.setDefaultFieldConjunction(searchParameters.getDefaultFTSOperator().equals(Operator.AND)); - parser.setMode(searchParameters.getDefaultFTSOperator().equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); - CommonTree ftsNode = (CommonTree) parser.ftsQuery().getTree(); + private ShardInstance detach(ShardInstance shardNode) { + ShardInstance detachedShardNode = new ShardInstance(); + detachedShardNode.setHostName(shardNode.getHostName()); + detachedShardNode.setPort(shardNode.getPort()); + detachedShardNode.setBaseUrl(shardNode.getBaseUrl()); + return detachedShardNode; } - private SearchTerm extractCmisPropertySearchTerm(SearchParameters searchParameters, String field, String operator) { - // TODO include filter and other possible constraints - - if (searchParameters.getQuery() == null) - return null; - - CharStream cs = new ANTLRStringStream(searchParameters.getQuery()); - CMISLexer lexer = new CMISLexer(); - CommonTokenStream tokens = new CommonTokenStream(lexer); - FTSParser parser = new FTSParser(tokens); - parser.setDefaultFieldConjunction(searchParameters.getDefaultFTSOperator().equals(Operator.AND)); - parser.setMode(searchParameters.getDefaultFTSOperator().equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); - CommonTree ftsNode = (CommonTree) parser.ftsQuery().getTree(); + private ShardState detach(ShardState shardState) { + ShardState detachedShardState = new ShardState(); + detachedShardState.setLastIndexedChangeSetCommitTime(shardState.getLastIndexedChangeSetCommitTime()); + detachedShardState.setLastIndexedChangeSetId(shardState.getLastIndexedChangeSetId()); + detachedShardState.setLastIndexedTxCommitTime(shardState.getLastIndexedTxCommitTime()); + detachedShardState.setLastIndexedTxId(shardState.getLastIndexedTxId()); + detachedShardState.setLastUpdated(shardState.getLastUpdated()); + detachedShardState.setMaster(shardState.isMaster()); + detachedShardState.setPropertyBag(shardState.getPropertyBag()); + return detachedShardState; } + private boolean putPutAdd(SimpleCache>> cache, K1 cacheKey, K2 mapKey, V mapValue) { + Map> map = cache.get(cacheKey); + if (map == null) + map = new HashMap<>(); + return this.putAdd(map, mapKey, mapValue); + } - - private class SearchTerm { - - private String field; - private String operator; - private String value; - + private boolean putAdd(Map> map, K key, V value) { + Set set = map.get(key); + if (set == null) + set = new HashSet<>(); + return set.add(value); } } diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties index ef89240..b3963b8 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties @@ -4,10 +4,22 @@ inteligr8.asie.idleShardExpirationInSeconds=${} +# Overrides of alfresco-repository.jar/alfresco/caches.properties +cache.shardStateSharedCache.tx.maxItems=16384 +cache.shardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.shardStateSharedCache.maxItems=16384 +cache.shardStateSharedCache.timeToLiveSeconds=1800 +cache.shardStateSharedCache.maxIdleSeconds=0 +cache.shardStateSharedCache.cluster.type=fully-distributed +cache.shardStateSharedCache.backup-count=1 +cache.shardStateSharedCache.eviction-policy=LRU +cache.shardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy +cache.shardStateSharedCache.readBackupData=false + # maxItems needs to be greater than total shards, including HA instances -cache.offlineShardStateSharedCache.tx.maxItems=1024 +cache.offlineShardStateSharedCache.tx.maxItems=16384 cache.offlineShardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.offlineShardStateSharedCache.maxItems=1024 +cache.offlineShardStateSharedCache.maxItems=16384 cache.offlineShardStateSharedCache.timeToLiveSeconds=1800 cache.offlineShardStateSharedCache.maxIdleSeconds=0 cache.offlineShardStateSharedCache.cluster.type=fully-distributed @@ -16,9 +28,9 @@ cache.offlineShardStateSharedCache.eviction-policy=LRU cache.offlineShardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy cache.offlineShardStateSharedCache.readBackupData=false -cache.coreExplicitIdSharedCache.tx.maxItems=1024 +cache.coreExplicitIdSharedCache.tx.maxItems=16384 cache.coreExplicitIdSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.coreExplicitIdSharedCache.maxItems=1024 +cache.coreExplicitIdSharedCache.maxItems=16384 cache.coreExplicitIdSharedCache.timeToLiveSeconds=1800 cache.coreExplicitIdSharedCache.maxIdleSeconds=0 cache.coreExplicitIdSharedCache.cluster.type=fully-distributed diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties index 6c345f1..e69de29 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties @@ -1,3 +0,0 @@ - -logger.inteligr8-asie.name=com.inteligr8.alfresco.asie -logger.inteligr8-asie.level=INFO diff --git a/community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java b/community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java new file mode 100644 index 0000000..1e05d44 --- /dev/null +++ b/community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java @@ -0,0 +1,146 @@ +package com.inteligr8.alfresco.asie; + +import java.util.ArrayList; +import java.util.List; + +import org.alfresco.repo.search.impl.parsers.FTSLexer; +import org.alfresco.repo.search.impl.parsers.FTSParser; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; + +public class QueryConstraintUnitTest { + + private static final ObjectMapper om = new ObjectMapper(); + + @BeforeClass + public static void init() { + SimpleModule module = new SimpleModule(); + module.addSerializer(Tree.class, new TreeSerializer()); + om.registerModule(module); + } + + @Test + public void testSingleExactTerm() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("=@cm:title:test", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "EXACT_TERM", "test"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleFuzzyTerm() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("@cm:title:test", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "TERM", "test"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleFuzzyString() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("@cm:title:'testing'", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "PHRASE", "'testing'"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleFuzzyStringDoubleQuotes() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("cm:title:\"testing\"", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "PHRASE", "\"testing\""); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleRange() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("@cm:created:[NOW TO '2025-01-01T00:00:00'>", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "RANGE", "INCLUSIVE", "NOW", "'2025-01-01T00:00:00'", "EXCLUSIVE"); + tree = this.validateChildren(tree, "FIELD_REF", "created"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testTwoTerms() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("=@cm:title:test1 AND @cm:author:test2", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + List trees = this.validateChildren(tree, "CONJUNCTION", 2); + + tree = trees.get(0); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "EXACT_TERM", "test1"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + + tree = trees.get(1); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "TERM", "test2"); + tree = this.validateChildren(tree, "FIELD_REF", "author"); + this.validate(tree, "PREFIX", "cm"); + } + + protected void validate(Tree tree, String text, String... extraValues) { + Assert.assertNotNull(tree); + Assert.assertEquals(text, tree.getText()); + Assert.assertEquals(extraValues.length, tree.getChildCount()); + for (int c = 0; c < extraValues.length; c++) + Assert.assertEquals(extraValues[c], tree.getChild(c).getText()); + } + + protected Tree validateChildren(Tree tree, String text, String... extraValues) { + Assert.assertNotNull(tree); + Assert.assertEquals(text, tree.getText()); + Assert.assertEquals(extraValues.length + 1, tree.getChildCount()); + for (int c = 0; c < extraValues.length; c++) + Assert.assertEquals(extraValues[c], tree.getChild(c).getText()); + return tree.getChild(extraValues.length); + } + + protected List validateChildren(Tree tree, String text, int count) { + Assert.assertNotNull(tree); + Assert.assertEquals(text, tree.getText()); + Assert.assertEquals(count, tree.getChildCount()); + List children = new ArrayList<>(); + for (int c = 0; c < tree.getChildCount(); c++) + children.add(tree.getChild(c)); + return children; + } + + protected Tree parseFts(String ftsQuery, Operator defaultOperator) throws RecognitionException, JsonProcessingException { + CharStream cs = new ANTLRStringStream(ftsQuery); + FTSLexer lexer = new FTSLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + FTSParser parser = new FTSParser(tokens); + parser.setDefaultFieldConjunction(defaultOperator.equals(Operator.AND)); + parser.setMode(defaultOperator.equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); + CommonTree tree = (CommonTree) parser.ftsQuery().getTree(); + return tree; + } + +} diff --git a/community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java b/community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java new file mode 100644 index 0000000..b9da03a --- /dev/null +++ b/community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java @@ -0,0 +1,44 @@ +package com.inteligr8.alfresco.asie; + +import java.io.IOException; + +import org.antlr.runtime.tree.Tree; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +public class TreeSerializer extends StdSerializer { + + private static final long serialVersionUID = -2714782538361726878L; + + public TreeSerializer() { + super(Tree.class); + } + + public TreeSerializer(Class type) { + super(type); + } + + public TreeSerializer(JavaType type) { + super(type); + } + + @Override + public void serialize(Tree value, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeStartObject(); + if (value.getText() != null) + gen.writeStringField("text", value.getText()); + + if (value.getChildCount() > 0) { + gen.writeArrayFieldStart("children"); + for (int c = 0; c < value.getChildCount(); c++) + gen.writeObject(value.getChild(c)); + gen.writeEndArray(); + } + + gen.writeEndObject(); + } + +} From de8e0bf2d73af4c8fa4395bcf6d4efac2214c15c Mon Sep 17 00:00:00 2001 From: "Brian M. Long" Date: Wed, 13 Nov 2024 18:03:22 -0500 Subject: [PATCH 4/5] update from refactoring (incomplete) --- community-module/pom.xml | 5 + .../alfresco/asie/CommunityConstants.java | 23 + .../alfresco/asie/cache/MultiValueCache.java | 67 -- .../asie/service/ShardStateService.java | 63 +- .../asie/service/SolrShardRegistry.java | 607 +++++++++++------- .../asie/util/ShardSetSearchComparator.java | 79 +++ 6 files changed, 490 insertions(+), 354 deletions(-) create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java delete mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java diff --git a/community-module/pom.xml b/community-module/pom.xml index 359c4f3..9ee7607 100644 --- a/community-module/pom.xml +++ b/community-module/pom.xml @@ -33,6 +33,11 @@ + + com.inteligr8.alfresco + cachext-platform-module + 1.0-SNAPSHOT + com.inteligr8.alfresco asie-shared diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java new file mode 100644 index 0000000..ce16419 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java @@ -0,0 +1,23 @@ +package com.inteligr8.alfresco.asie; + +public interface CommunityConstants extends Constants { + + static final String BEAN_SHARDSETS_CACHE = "asieShardsetsCache"; + static final String BEAN_NODES_CACHE = "asieNodesCache"; + static final String BEAN_SHARD_NODES_CACHE = "asieShardNodesCache"; + static final String BEAN_SHARDINST_STATE_CACHE = "asieShardInstanceStateCache"; + static final String BEAN_NODE_DISABLE_CACHE = "asieNodeDisabledCache"; + static final String BEAN_NODE_UNAVAIL_CACHE = "asieNodeUnavailableCache"; + static final String BEAN_SHARDINST_DISABLE_CACHE = "asieShardInstanceDisabledCache"; + static final String BEAN_SHARDINST_UNAVAIL_CACHE = "asieShardInstanceUnavailableCache"; + static final String BEAN_CORE_EXPLICIT_CACHE = "asieCoreExplicitCache"; + + static final String ATTR_ASIE_SHARDSET = "inteligr8.asie.shardSet"; + static final String ATTR_ASIE_NODE = "inteligr8.asie.node"; + static final String ATTR_ASIE_SHARD_NODES = "inteligr8.asie.shard.nodes"; + static final String ATTR_ASIE_SHARD_NODE = "inteligr8.asie.shard.node"; + static final String ATTR_OBJECT = "object"; + static final String ATTR_DISABLE = "disabled"; + static final String ATTR_NODES = "nodes"; + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java deleted file mode 100644 index d6d820c..0000000 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/cache/MultiValueCache.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.inteligr8.alfresco.asie.cache; - -import java.io.Serializable; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Collection; - -import org.alfresco.repo.cache.SimpleCache; - -public class MultiValueCache> implements SimpleCache { - - private SimpleCache cache; - private Class collectionType; - - public MultiValueCache(SimpleCache cache, Class collectionType) { - this.cache = cache; - this.collectionType = collectionType; - } - - @SuppressWarnings("unchecked") - public boolean add(K key, V value) { - C c = this.cache.get(key); - if (c != null) - return c.add(value); - - try { - Constructor constructor = this.collectionType.getConstructor(); - c = (C) constructor.newInstance(); - this.cache.put(key, c); - return c.add(value); - } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException | InstantiationException e) { - throw new UnsupportedOperationException(e.getMessage(), e); - } - } - - @Override - public void clear() { - this.cache.clear(); - } - - @Override - public C get(K key) { - return this.cache.get(key); - } - - @Override - public boolean contains(K key) { - C c = this.cache.get(key); - return c == null ? false : !c.isEmpty(); - } - - @Override - public Collection getKeys() { - return this.cache.getKeys(); - } - - @Override - public void put(K key, C value) { - this.cache.put(key, value); - } - - @Override - public void remove(K key) { - this.cache.remove(key); - } - -} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java index c93c5ef..5a43bf5 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java @@ -1,80 +1,25 @@ package com.inteligr8.alfresco.asie.service; -import java.io.Serializable; -import java.util.Arrays; - -import org.alfresco.repo.cache.SimpleCache; -import org.alfresco.repo.index.shard.ShardInstance; -import org.alfresco.repo.index.shard.ShardState; import org.alfresco.service.cmr.attributes.AttributeService; -import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; -import org.apache.commons.lang3.ArrayUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import com.inteligr8.alfresco.asie.Constants; -import com.inteligr8.alfresco.asie.enterprise.EnterpriseConstants; @Component public class ShardStateService implements com.inteligr8.alfresco.asie.spi.ShardStateService { - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - @Autowired @Qualifier(Constants.QUALIFIER_ASIE) private AttributeService attrService; - + @Autowired - @Qualifier(Constants.BEAN_SHARD_STATE_CACHE) - private SimpleCache shardStateCache; + private SolrShardRegistry shardRegistry; + @Override public void clear() { - this.logger.info("Removing all nodes/shards from the shard registry"); - - // this clears the state from the backend database - this.attrService.removeAttributes(EnterpriseConstants.ATTR_SHARD_STATE); - this.attrService.removeAttributes(EnterpriseConstants.ATTR_SHARD_SUBSCRIPTION); - this.attrService.removeAttributes(Constants.ATTR_ASIE); - - // this clears the state from Hazelcast - this.shardStateCache.clear(); - this.shardToGuidCache.clear(); - } - - public void remove(Serializable... keys) { - if (keys.length == 0) - throw new IllegalArgumentException(); - - this.logger.info("Removing from the shard registry: {}", Arrays.toString(keys)); - - Serializable[] shardStateKeys = keys; - Serializable[] shardSubKeys; - if (EnterpriseConstants.ATTR_SHARD_STATE.equals(keys[0])) { - shardSubKeys = ArrayUtils.clone(keys); - shardSubKeys[0] = EnterpriseConstants.ATTR_SHARD_SUBSCRIPTION; - } else { - shardStateKeys = ArrayUtils.addFirst(keys, EnterpriseConstants.ATTR_SHARD_STATE); - shardSubKeys = ArrayUtils.addFirst(keys, EnterpriseConstants.ATTR_SHARD_SUBSCRIPTION); - } - - ShardState shardState = (ShardState) this.attrService.getAttribute(shardStateKeys); - - // this clears the state from the backend database - this.attrService.removeAttribute(shardStateKeys); - this.attrService.removeAttribute(shardSubKeys); - - // this clears the state from Hazelcast - if (shardState != null) { - this.shardStateCache.remove(shardState.getShardInstance()); - this.shardToGuidCache.remove(shardState.getShardInstance()); - } - } - - public void iterate(AttributeQueryCallback callback) { - this.attrService.getAttributes(callback, EnterpriseConstants.ATTR_SHARD_STATE); + this.shardRegistry.purge(); } } diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java index c982712..52381ae 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java @@ -1,6 +1,7 @@ package com.inteligr8.alfresco.asie.service; import java.io.Serializable; +import java.time.OffsetDateTime; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -8,18 +9,12 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.OptionalInt; +import java.util.Random; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.alfresco.repo.cache.SimpleCache; import org.alfresco.repo.index.shard.Floc; -import org.alfresco.repo.index.shard.Shard; -import org.alfresco.repo.index.shard.ShardInstance; -import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.alfresco.repo.index.shard.ShardRegistry; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.lock.JobLockService; import org.alfresco.service.cmr.attributes.AttributeService; @@ -29,7 +24,6 @@ import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.search.SearchParameters; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; -import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -39,27 +33,29 @@ import org.springframework.context.ApplicationEvent; import org.springframework.extensions.surf.util.AbstractLifecycleBean; import org.springframework.stereotype.Component; +import com.inteligr8.alfresco.asie.CommunityConstants; import com.inteligr8.alfresco.asie.Constants; -import com.inteligr8.alfresco.asie.cache.MultiValueCache; import com.inteligr8.alfresco.asie.compute.QueryInspector; import com.inteligr8.alfresco.asie.compute.QueryInspector.QueryRangeValue; import com.inteligr8.alfresco.asie.compute.QueryInspector.QuerySingleValue; import com.inteligr8.alfresco.asie.compute.QueryInspector.QueryValue; import com.inteligr8.alfresco.asie.compute.QueryInspectorFactory; -import com.inteligr8.alfresco.asie.model.Node; +import com.inteligr8.alfresco.asie.model.Shard; +import com.inteligr8.alfresco.asie.model.ShardInstance; +import com.inteligr8.alfresco.asie.model.ShardInstanceState; import com.inteligr8.alfresco.asie.model.ShardSet; +import com.inteligr8.alfresco.asie.model.SolrHost; +import com.inteligr8.alfresco.asie.spi.ShardRegistry; +import com.inteligr8.alfresco.cachext.CollectionCache; +import com.inteligr8.alfresco.cachext.MultiValueCache; @Component public class SolrShardRegistry extends AbstractLifecycleBean implements ShardRegistry { private final Logger logger = LoggerFactory.getLogger(this.getClass()); - private final Pattern coreShardPattern = Pattern.compile("(.+)-[0-9]+"); - + private final Random random = new Random(); private final QName shardLock = QName.createQName(Constants.NAMESPACE_ASIE, "shardLock"); - @Autowired - private ShardStateService sss; - @Autowired @Qualifier(Constants.QUALIFIER_ASIE) private AttributeService attrService; @@ -77,19 +73,39 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg private JobLockService jobLockService; @Autowired - @Qualifier(Constants.BEAN_FLOC_SHARD_NODE_CACHE) - private SimpleCache>> flocShardNodeCache; + @Qualifier(CommunityConstants.BEAN_SHARDSETS_CACHE) + private SimpleCache shardsetsCache; @Autowired - @Qualifier(Constants.BEAN_ONLINE_SHARD_STATE_CACHE) - private SimpleCache onlineNodeShardStateCache; + @Qualifier(CommunityConstants.BEAN_NODES_CACHE) + private SimpleCache nodesCache; @Autowired - @Qualifier(Constants.BEAN_OFFILINE_SHARD_STATE_CACHE) - private SimpleCache offlineNodeShardStateCache; + @Qualifier(CommunityConstants.BEAN_SHARD_NODES_CACHE) + private MultiValueCache shardNodesCache; @Autowired - @Qualifier(Constants.BEAN_CORE_EXPLICIT_CACHE) + @Qualifier(CommunityConstants.BEAN_SHARDINST_STATE_CACHE) + private SimpleCache shardInstanceStatesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_UNAVAIL_CACHE) + private CollectionCache> nodeUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_DISABLE_CACHE) + private CollectionCache> nodeDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_UNAVAIL_CACHE) + private CollectionCache> shardInstanceUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_DISABLE_CACHE) + private CollectionCache> shardInstanceDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) private SimpleCache coreExplicitIdCache; @Value("${inteligr8.asie.registerUnknownShardOffline}") @@ -103,204 +119,343 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg @Override protected void onBootstrap(ApplicationEvent event) { - String lock = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); - try { - this.attrService.getAttributes(new AttributeQueryCallback() { - @Override - public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { - switch ((String) keys[2]) { - case Constants.ATTR_STATE: - ShardState shardNodeState = (ShardState) value; - ShardInstance shardNode = shardNodeState.getShardInstance(); - cacheShard(shardNode, shardNodeState, (String) keys[1]); - return true; - default: - return true; - - } - } - }, Constants.ATTR_ASIE_NODE_SHARD); - } finally { - this.jobLockService.releaseLock(lock, this.shardLock); - } + this.loadPersistedToCache(); } @Override protected void onShutdown(ApplicationEvent event) { } - /** - * This is private because it must be wrapped in a cluster-safe lock - */ - private void cacheShard(ShardInstance shardNode, ShardState shardNodeState, String nodeShardId) { - ShardInstance detachedShardNode = this.detach(shardNode); - - SimpleCache shardCache = this.onlineNodeShardStateCache; - ShardState cachedShardNodeState = this.onlineNodeShardStateCache.get(detachedShardNode); - if (cachedShardNodeState == null) { - cachedShardNodeState = this.offlineNodeShardStateCache.get(detachedShardNode); - shardCache = this.offlineNodeShardStateCache; - } + protected void loadPersistedToCache() { + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + String core = (String) keys[1]; + if (!shardsetsCache.contains(core)) { + ShardSet shardSet = (ShardSet) value; + shardsetsCache.put(core, shardSet); + + switch (shardSet.getMethod()) { + case EXPLICIT_ID: + cacheExplicitShard(shardSet, false); + break; + default: + } + } + return true; + } + }, CommunityConstants.ATTR_ASIE_SHARDSET); - Shard shard = shardNode.getShard(); - this.putPutAdd(this.flocShardNodeCache, shard.getFloc(), shard.getInstance(), detachedShardNode); - - if (cachedShardNodeState == null) { - Boolean online = (Boolean) this.attrService.getAttribute(Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_ONLINE); - if (online != null) { - if (online.booleanValue()) { - this.onlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); - } else { - this.offlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + String nodeSpec = (String) keys[2]; + SolrHost node = (SolrHost) value; + if (!nodesCache.contains(nodeSpec)) + nodesCache.put(nodeSpec, node); + if (Boolean.TRUE.equals(attrService.getAttribute(CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE, nodeSpec))) { + if (!nodeDisableCache.contains(node)) + nodeDisableCache.add(node); + } else if (nodeDisableCache.contains(node)) { + nodeDisableCache.remove(node); + } + return true; } - } else { - if (this.registerOffline) { - this.offlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); - } else { - this.onlineNodeShardStateCache.put(detachedShardNode, cachedShardNodeState); + }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT); + + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + Shard shard = (Shard) keys[1]; + SolrHost node = (SolrHost) keys[2]; + if (!shardNodesCache.contains(shard, node)) + shardNodesCache.add(shard, node); + return true; } - } - } else if (cachedShardNodeState.getLastIndexedTxId() < shardNodeState.getLastIndexedTxId()) { - // update the cached state if the state's last indexes transaction is later - shardCache.put(shardNode, this.detach(shardNodeState)); - } - - switch (shardNode.getShard().getFloc().getShardMethod()) { - case EXPLICIT_ID: - cacheExplicitShard(shardNode, shardNodeState); - break; - default: + }, CommunityConstants.ATTR_ASIE_SHARD_NODES); + + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + ShardInstance shardNode = (ShardInstance) keys[2]; + ShardInstanceState state = (ShardInstanceState) value; + if (!shardInstanceStatesCache.contains(shardNode)) + shardInstanceStatesCache.put(shardNode, state); + if (Boolean.TRUE.equals(attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE, shardNode))) { + if (!shardInstanceDisableCache.contains(shardNode)) + shardInstanceDisableCache.add(shardNode); + } else if (shardInstanceDisableCache.contains(shardNode)) { + shardInstanceDisableCache.remove(shardNode); + } + return true; + } + }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); } } - private void cacheExplicitShard(ShardInstance shardNode, ShardState shardNodeState) { - String coreName = shardNode.getShard().getFloc().getPropertyBag().get("coreName"); - if (coreName != null && !this.coreExplicitIdCache.contains(coreName)) { - String property = shardNodeState.getPropertyBag().get("shard.key"); - QName propertyQName = QName.createQName(property, this.namespaceService); - - this.logger.debug("Mapping core to explicit ID: {} => {}", coreName, propertyQName); - this.coreExplicitIdCache.put(coreName, propertyQName); + private void cacheExplicitShard(ShardSet shardSet, boolean overwrite) { + if (overwrite || !this.coreExplicitIdCache.contains(shardSet.getCore())) { + String property = shardSet.getPrefixedProperty(); + QName propertyQName = QName.createQName(property, namespaceService); + + this.logger.debug("Mapping core to explicit ID: {} => {}", shardSet.getCore(), propertyQName); + this.coreExplicitIdCache.put(shardSet.getCore(), propertyQName); } } + protected void persistCache() { + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 100L, 50); + try { + this.persistShardSetCache(); + this.persistNodeCache(); + this.persistShardNodeCache(); + this.persistShardInstanceCache(); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + private void persistShardSetCache() { + // add anything missing + // update anything changed + for (String core : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(core); + this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, core); + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + } + + private void persistNodeCache() { + // add anything missing + // update anything changed + for (String nodeSpec : this.nodesCache.getKeys()) { + SolrHost node = this.nodesCache.get(nodeSpec); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, nodeSpec); + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + + // add anything disabled + for (SolrHost node : this.nodeDisableCache.values()) + this.checkSetAttribute(Boolean.TRUE, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE, node.getSpec()); + + // remove anything not disabled + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + SolrHost node = SolrHost.from((String) keys[2]); + if (!nodeDisableCache.contains(node)) + attrService.removeAttribute(keys); + return true; + } + }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE); + } + + private void persistShardNodeCache() { + // add anything missing + // update anything changed + for (Shard shard : this.shardNodesCache.getKeys()) { + Collection nodes = this.shardNodesCache.get(shard); + for (SolrHost node : nodes) { + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + } + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + } + + private void persistShardInstanceCache() { + // add anything missing + // update anything changed + for (ShardInstance shardNode : this.shardInstanceStatesCache.getKeys()) { + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + ShardInstanceState currentState = (ShardInstanceState) this.attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + if (currentState != null) { + if (currentState.compareTo(state) >= 0) { + // do nothing + } else { + this.logger.debug("The persisted state was old; updating: {}: {} => {}", shardNode, currentState, state); + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } + } else { + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + + // add anything disabled + for (ShardInstance shardNode : this.shardInstanceDisableCache.values()) + this.checkSetAttribute(Boolean.TRUE, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE, shardNode); + + // remove anything not disabled + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + ShardInstance shardNode = (ShardInstance) keys[2]; + if (!shardInstanceDisableCache.contains(shardNode)) + attrService.removeAttribute(keys); + return true; + } + }, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE); + } + + protected void persistCache(ShardSet shardSet, SolrHost node, Shard shard, ShardInstance shardNode, ShardInstanceState state) { + String lockId = this.jobLockService.getLock(this.shardLock, 2000L, 100L, 50); + try { + this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, shardSet.getCore()); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, node.getSpec()); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + this.checkSetAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + private void checkSetAttribute(Serializable value, Serializable... keys) { + Serializable currentValue = this.attrService.getAttribute(keys); + if (currentValue != null) { + if (currentValue.equals(value)) + return; + this.logger.warn("The attribute value unexpectedly changed: {}: {} => {}", keys, currentValue, value); + } + + this.attrService.setAttribute(value, keys); + } + @Override public void registerShardState(ShardState shardNodeState) { - ShardInstance shardNode = shardNodeState.getShardInstance(); - Node node = new Node(shardNode); - this.fixFlocPropertyBag(shardNodeState); + ShardSet shardSet = ShardSet.from(shardNodeState.getShardInstance().getShard().getFloc(), shardNodeState); + Shard shard = Shard.from(shardSet, shardNodeState.getShardInstance().getShard().getInstance()); + SolrHost node = SolrHost.from(shardNodeState.getShardInstance()); + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = ShardInstanceState.from(shardNodeState); - String lock = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); - try { - this.cacheShard(shardNode, shardNodeState, node.getId()); - this.persistShards(); - } finally { - this.jobLockService.releaseLock(lock, this.shardLock); - } - } - - protected void fixFlocPropertyBag(ShardState shardNodeState) { - Floc floc = shardNodeState.getShardInstance().getShard().getFloc(); - if (floc.getPropertyBag().isEmpty()) { - for (Entry prop : shardNodeState.getPropertyBag().entrySet()) { - if (prop.getKey().startsWith("shard.")) { - floc.getPropertyBag().put(prop.getKey(), prop.getValue()); - } else if (prop.getKey().equals("coreName")) { - String coreName = this.extractCoreName(prop.getValue()); - if (coreName != null) - floc.getPropertyBag().put(prop.getKey(), coreName); - } - } - } - } - - protected String extractCoreName(String coreShardName) { - Matcher matcher = this.coreShardPattern.matcher(coreShardName); - if (!matcher.matches()) - return null; - return matcher.group(1); - } - - /** - * This is private because it must be wrapped in a cluster-safe lock - */ - private void persistShards() { - long onlineExpired = System.currentTimeMillis() - this.offlineIdleShardInSeconds * 1000L; - long offlineExpired = System.currentTimeMillis() - this.forgetOfflineShardInSeconds * 1000L; - - for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { - String nodeShardId = new Node(shardNode).getId() + ";" + shardNode.getShard().getInstance(); - ShardState shardNodeState = this.onlineNodeShardStateCache.get(shardNode); - if (shardNodeState.getLastUpdated() < onlineExpired) { - this.logger.warn("Taking shard offline: {}", shardNode); - this.onlineNodeShardStateCache.remove(shardNode); - this.offlineNodeShardStateCache.put(shardNode, shardNodeState); - } else { - this.attrService.setAttribute(shardNodeState, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_STATE); - this.attrService.setAttribute(Boolean.TRUE, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_ONLINE); - } - } - - for (ShardInstance shardNode : this.offlineNodeShardStateCache.getKeys()) { - String nodeShardId = new Node(shardNode).getId() + ";" + shardNode.getShard().getInstance(); - ShardState shardNodeState = this.offlineNodeShardStateCache.get(shardNode); - if (shardNodeState.getLastUpdated() < offlineExpired) { - this.logger.info("Forgetting about already offline shard: {}", shardNode); - this.offlineNodeShardStateCache.remove(shardNode); - } else { - this.attrService.setAttribute(shardNodeState, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_STATE); - this.attrService.setAttribute(Boolean.FALSE, Constants.ATTR_ASIE_NODE_SHARD, nodeShardId, Constants.ATTR_ONLINE); - } - } + this.persistCache(shardSet, node, shard, shardNode, state); } @Override - public Map>> getFlocs() { - Map>> flocs = new HashMap<>(); + public void unregisterShardInstance(org.alfresco.repo.index.shard.ShardInstance shardInstance) { + ShardSet shardSet = ShardSet.from(shardInstance.getShard().getFloc(), null); + Shard shard = Shard.from(shardSet, shardInstance.getShard().getInstance()); + SolrHost node = SolrHost.from(shardInstance); + ShardInstance shardNode = ShardInstance.from(shard, node); + + this.shardInstanceStatesCache.remove(shardNode); + this.shardInstanceDisableCache.remove(shardNode); + this.shardInstanceUnavailableCache.remove(shardNode); + this.nodeDisableCache.remove(node); + this.nodeUnavailableCache.remove(node); + this.attrService.removeAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + } + + @Override + public Map>> getFlocs() { + Map flocs = new HashMap<>(); + Map>> response = new HashMap<>(); - for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { - Floc floc = shardNode.getShard().getFloc(); + for (Shard shard : this.shardNodesCache.getKeys()) { + String core = shard.extractShardSetCore(); + ShardSet shardSet = this.shardsetsCache.get(core); + + Map> shards; + Floc floc = flocs.get(core); + if (floc != null) { + floc = shardSet.toAlfrescoModel(); + shards = new HashMap<>(); + } else { + shards = response.get(floc); + } - Map> shards = flocs.get(floc); - if (shards == null) - flocs.put(floc, shards = new HashMap<>()); + org.alfresco.repo.index.shard.Shard shard_ = shard.toAlfrescoModel(floc); + Set states = shards.get(shard_); + if (states == null) + states = new HashSet<>(); - Set shardNodeStates = shards.get(shardNode.getShard()); - if (shardNodeStates == null) - shards.put(shardNode.getShard(), shardNodeStates = new HashSet<>()); + for (SolrHost node : this.shardNodesCache.get(shard)) { + if (this.nodeDisableCache.contains(node) || this.nodeUnavailableCache.contains(node)) { + this.logger.debug("Excluding node as it is disabled or considered unavailable: {}", node); + continue; + } + + ShardInstance shardNode = ShardInstance.from(shard, node); + if (this.shardInstanceDisableCache.contains(shardNode) || this.shardInstanceUnavailableCache.contains(shardNode)) { + this.logger.debug("Excluding shard node as it is disabled or considered unavailable: {}", shardNode); + continue; + } + + ShardInstanceState shardNodeState = this.shardInstanceStatesCache.get(shardNode); + states.add(shardNodeState.toAlfrescoModel(shardNode.toAlfrescoModel(shard_))); + } - ShardState shardNodeState = this.onlineNodeShardStateCache.get(shardNode); - if (shardNodeState != null) // in case it was removed during the looping (very rare) - shardNodeStates.add(shardNodeState); + if (!states.isEmpty()) + shards.put(shard_, states); + if (!shards.isEmpty()) + response.put(floc, shards); } - return flocs; + return response; } @Override public void purge() { - this.sss.clear(); + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 100L, 50); + try { + this.logger.info("Removing all nodes/shards from the shard registry"); + this.shardsetsCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARDSET); + + this.nodesCache.clear(); + this.nodeDisableCache.clear(); + this.nodeUnavailableCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_NODE); + + this.shardNodesCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARD_NODES); + + this.shardInstanceStatesCache.clear(); + this.shardInstanceDisableCache.clear(); + this.shardInstanceUnavailableCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARD_NODE); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } } @Override public void purgeAgedOutShards() { - long onlineExpired = System.currentTimeMillis() - this.offlineIdleShardInSeconds * 1000L; - long offlineExpired = System.currentTimeMillis() - this.forgetOfflineShardInSeconds * 1000L; + OffsetDateTime onlineExpired = OffsetDateTime.now().minusSeconds(this.offlineIdleShardInSeconds); + OffsetDateTime offlineExpired = OffsetDateTime.now().minusSeconds(this.forgetOfflineShardInSeconds); - for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { - ShardState shardNodeState = this.onlineNodeShardStateCache.get(shardNode); - if (shardNodeState.getLastUpdated() < onlineExpired) { + for (ShardInstance shardNode : this.shardInstanceStatesCache.getKeys()) { + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + SolrHost node = shardNode.extractNode(); + + if (this.shardInstanceDisableCache.contains(shardNode)) { + this.logger.debug("Ignoring disabled shard instance during purgeAgedOutShards()"); + } else if (this.nodeDisableCache.contains(node)) { + this.logger.debug("Ignoring disabled node during purgeAgedOutShards()"); + } else if (state.getLastUpdated().isBefore(offlineExpired)) { + this.shardInstanceStatesCache.remove(shardNode); + if (this.shardInstanceUnavailableCache.remove(shardNode)) { + this.logger.info("Forgetting about already offline shard: {}", shardNode); + } else if (this.nodeUnavailableCache.remove(node)) { + this.logger.info("Forgetting about already offline shard: {}", shardNode); + } else { + this.logger.warn("Forgetting about online shard: {}", shardNode); + } + } else if (state.getLastUpdated().isBefore(onlineExpired)) { this.logger.warn("Taking shard offline: {}", shardNode); - this.onlineNodeShardStateCache.remove(shardNode); - this.offlineNodeShardStateCache.put(shardNode, shardNodeState); - } - } - - for (ShardInstance shardNode : this.offlineNodeShardStateCache.getKeys()) { - ShardState shardNodeState = this.offlineNodeShardStateCache.get(shardNode); - if (shardNodeState.getLastUpdated() < offlineExpired) { - this.logger.info("Forgetting about already offline shard: {}", shardNode); - this.offlineNodeShardStateCache.remove(shardNode); + this.shardInstanceUnavailableCache.add(shardNode); } } } @@ -314,8 +469,15 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg public Set getShardInstanceList(String coreName) { Set shardIds = new HashSet<>(); - for (ShardInstance shardNode : this.onlineNodeShardStateCache.getKeys()) { - shardIds.add(shardNode.getShard().getInstance()); + ShardSet shardSet = this.shardsetsCache.get(coreName); + if (shardSet == null) + return Collections.emptySet(); + + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore())) { + shardIds.add(shard.extractShardId()); + } } return shardIds; @@ -327,18 +489,19 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg } @Override - public List getIndexSlice(SearchParameters searchParameters) { + public List getIndexSlice(SearchParameters searchParameters) { if (searchParameters.getQuery() == null) return Collections.emptyList(); - List bestShards = null; + List bestShards = null; - for (Floc floc : this.flocShardMultiCache.getKeys()) { - List shards = new LinkedList<>(); + for (String shardSetSpec : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(shardSetSpec); + List shards = new LinkedList<>(); - switch (floc.getShardMethod()) { + switch (shardSet.getMethod()) { case EXPLICIT_ID: - String property = floc.getPropertyBag().get("shard.key"); + String property = shardSet.getPrefixedProperty(); QName propertyQName = QName.createQName(property, this.namespaceService); DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); @@ -367,8 +530,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg } } - // shardIds to shardInstances - break; + shards.addAll(this.getIndexSlice(shardSet, shardIds)); default: // make no determination } @@ -380,42 +542,31 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg return bestShards; } - protected List getIndexSlice() { + protected List getIndexSlice(ShardSet shardSet, Collection shardIds) { + List shardNodes = new LinkedList<>(); - } - - private ShardInstance detach(ShardInstance shardNode) { - ShardInstance detachedShardNode = new ShardInstance(); - detachedShardNode.setHostName(shardNode.getHostName()); - detachedShardNode.setPort(shardNode.getPort()); - detachedShardNode.setBaseUrl(shardNode.getBaseUrl()); - return detachedShardNode; - } - - private ShardState detach(ShardState shardState) { - ShardState detachedShardState = new ShardState(); - detachedShardState.setLastIndexedChangeSetCommitTime(shardState.getLastIndexedChangeSetCommitTime()); - detachedShardState.setLastIndexedChangeSetId(shardState.getLastIndexedChangeSetId()); - detachedShardState.setLastIndexedTxCommitTime(shardState.getLastIndexedTxCommitTime()); - detachedShardState.setLastIndexedTxId(shardState.getLastIndexedTxId()); - detachedShardState.setLastUpdated(shardState.getLastUpdated()); - detachedShardState.setMaster(shardState.isMaster()); - detachedShardState.setPropertyBag(shardState.getPropertyBag()); - return detachedShardState; - } - - private boolean putPutAdd(SimpleCache>> cache, K1 cacheKey, K2 mapKey, V mapValue) { - Map> map = cache.get(cacheKey); - if (map == null) - map = new HashMap<>(); - return this.putAdd(map, mapKey, mapValue); - } - - private boolean putAdd(Map> map, K key, V value) { - Set set = map.get(key); - if (set == null) - set = new HashSet<>(); - return set.add(value); + for (Integer shardId : shardIds) { + Shard shard = Shard.from(shardSet, shardId); + + Collection nodes = this.shardNodesCache.get(shard); + List availableNodes = new LinkedList<>(); + for (SolrHost node : nodes) { + if (this.nodeDisableCache.contains(node) || this.nodeUnavailableCache.contains(node)) + continue; + + ShardInstance shardNode = ShardInstance.from(shard, node); + if (this.shardInstanceDisableCache.contains(shardNode) || this.shardInstanceUnavailableCache.contains(shardNode)) + continue; + + availableNodes.add(node); + } + + SolrHost randomNode = availableNodes.get(this.random.nextInt(availableNodes.size())); + + shardNodes.add(ShardInstance.from(shard, randomNode).toAlfrescoModel(shard.toAlfrescoModel(shardSet.toAlfrescoModel()))); + } + + return shardNodes; } } diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java new file mode 100644 index 0000000..78905c5 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java @@ -0,0 +1,79 @@ +package com.inteligr8.alfresco.asie.util; + +import java.util.Comparator; + +import org.alfresco.repo.index.shard.ShardMethodEnum; + +import com.inteligr8.alfresco.asie.model.ShardSet; + +public class ShardSetSearchComparator implements Comparator { + + @Override + public int compare(ShardSet ss1, ShardSet ss2) { + int compare = this.compare(ss1.getMethod(), ss2.getMethod()); + if (compare != 0) + return compare; + + return this.compare(ss1.getShards(), ss2.getShards()); + } + + private int compare(ShardMethodEnum method1, ShardMethodEnum method2) { + if (method1.equals(method2)) + return 0; + + switch (method1) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + return -1; + case PROPERTY: + case DATE: + switch (method2) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + return 1; + default: + return -1; + } + case ACL_ID: + case MOD_ACL_ID: + switch (method2) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + case PROPERTY: + case DATE: + return 1; + default: + return -1; + } + default: + switch (method2) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + case PROPERTY: + case DATE: + case ACL_ID: + case MOD_ACL_ID: + return 1; + default: + } + } + + return 0; + } + + private int compare(Short shards1, Short shards2) { + // the larger the shard count, the more shards that may need to be queried + // so prefer smaller shard counts + // no shard count (DB_ID_RANGE) should be treated as the worst (unlimited) + if (shards1 == null && shards2 == null) { + return 0; + } else if (shards1 == null) { + return 1; + } else if (shards2 == null) { + return -1; + } else { + return shards1.compareTo(shards2); + } + } + +} From af7c9e148ed4167eb0bd43a10ff314d2da3c96b2 Mon Sep 17 00:00:00 2001 From: "Brian M. Long" Date: Thu, 14 Nov 2024 11:01:49 -0500 Subject: [PATCH 5/5] compiling/running module (untested) --- community-module/metadata.keystore | Bin 0 -> 409 bytes community-module/pom.xml | 23 +- community-module/rad.sh | 8 +- .../asie/compute/CmisQueryInspector.java | 7 +- .../asie/compute/QueryInspectorFactory.java | 3 +- .../asie/service/ShardDiscoveryService.java | 240 ++++++++++++++++++ .../asie/service/SolrShardRegistry.java | 192 +++++++++----- .../alfresco-global.properties | 135 +++++++--- .../module-context.xml | 40 ++- .../module.properties | 1 + enterprise-module/metadata.keystore | Bin 0 -> 409 bytes enterprise-module/pom.xml | 8 +- enterprise-module/rad.sh | 8 +- shared/pom.xml | 2 +- 14 files changed, 544 insertions(+), 123 deletions(-) create mode 100644 community-module/metadata.keystore create mode 100644 community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java create mode 100644 enterprise-module/metadata.keystore diff --git a/community-module/metadata.keystore b/community-module/metadata.keystore new file mode 100644 index 0000000000000000000000000000000000000000..2c2a1d961273b300b1852d37d43e21676277feb5 GIT binary patch literal 409 zcmXqLVw}pv$ZXKWXwSx})#lOmotKfFaX}NK0ZS934p3MFh*eOfM1fL*Kw(}W=49iB z>f+&IWL?m>*`RR)vJlgP#sw^ma}6428Kl8YVdXW5G_XK$c?`5!L=N{!9_e#B605cle)lJ&&JDO?$*;d>UWc8v;Y=9$(Y z?eMAldWXglLl*;ixMMg)429$b7;+g>8A=!u8B%~4NE;|36o?v%un2`@=B6qbnj0A# zm>QXw8(SC}8Tc9)8t}rM#>B|Vz@jjH<;y>ndk;UBxGpCCrcU6NQrG?ks@Yqco;K

dmVXfd literal 0 HcmV?d00001 diff --git a/community-module/pom.xml b/community-module/pom.xml index 9ee7607..15bbafe 100644 --- a/community-module/pom.xml +++ b/community-module/pom.xml @@ -6,7 +6,7 @@ com.inteligr8.alfresco asie-platform-module-parent - 1.1-SNAPSHOT + 1.2-SNAPSHOT ../ @@ -16,8 +16,12 @@ ASIE Platform Module for ACS Community - 5.2.0 - 7.0.0 + 4.9.0 + 23.3.0 + 23.3.0.98 + 10-2.1 + + true @@ -42,7 +46,6 @@ com.inteligr8.alfresco asie-shared ${project.version} - provided @@ -51,6 +54,14 @@ alfresco-repository provided + + + + com.inteligr8.alfresco + cxf-jaxrs-platform-module + 1.3.1-acs-v23.3 + amp + @@ -74,6 +85,10 @@ true + + com.inteligr8.ootbee:beedk-acs-search-rad-tile:[1.1.6,2.0.0) + + com.inteligr8.ootbee:beedk-acs-platform-self-rad-tile:[1.1.6,2.0.0) com.inteligr8.ootbee:beedk-acs-platform-module-tile:[1.1.6,2.0.0) diff --git a/community-module/rad.sh b/community-module/rad.sh index 7cb0a80..3c7c2fa 100644 --- a/community-module/rad.sh +++ b/community-module/rad.sh @@ -1,22 +1,22 @@ #!/bin/sh discoverArtifactId() { - ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate` + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'` } rebuild() { echo "Rebuilding project ..." - mvn process-classes + mvn process-test-classes } start() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad process-classes + mvn -Drad process-test-classes } start_log() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad -Ddocker.showLogs process-classes + mvn -Drad -Ddocker.showLogs process-test-classes } stop() { diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java index 7b0486b..44ca0d7 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java @@ -1,10 +1,12 @@ package com.inteligr8.alfresco.asie.compute; +import java.util.List; import java.util.Set; import org.alfresco.repo.search.impl.parsers.CMISLexer; import org.alfresco.repo.search.impl.parsers.CMISParser; import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.search.SearchService; import org.alfresco.service.namespace.QName; import org.antlr.runtime.ANTLRStringStream; @@ -31,8 +33,9 @@ public class CmisQueryInspector implements QueryInspector { } @Override - public QueryValue findRequiredProperty(String cmisQuery, Operator defaultOperator, QName property) throws RecognitionException { - Tree tree = this.parseCmis(cmisQuery, defaultOperator); + public List findRequiredPropertyValues(String query, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException { + Tree tree = this.parseCmis(query, defaultOperator); + throw new UnsupportedOperationException(); } protected Tree parseCmis(String cmisQuery, Operator defaultOperator) throws RecognitionException { diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java index bad416d..2e7035c 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java @@ -1,5 +1,6 @@ package com.inteligr8.alfresco.asie.compute; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,7 +15,7 @@ public class QueryInspectorFactory implements InitializingBean { @Autowired private List inspectors; - private Map languageInspectorMap; + private Map languageInspectorMap = new HashMap<>(); @Override public void afterPropertiesSet() throws Exception { diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java new file mode 100644 index 0000000..5cbbaf1 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java @@ -0,0 +1,240 @@ +package com.inteligr8.alfresco.asie.service; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.alfresco.repo.cache.SimpleCache; +import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.service.namespace.QName; +import org.alfresco.util.Pair; +import org.alfresco.util.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.CommunityConstants; +import com.inteligr8.alfresco.asie.model.Shard; +import com.inteligr8.alfresco.asie.model.ShardInstance; +import com.inteligr8.alfresco.asie.model.ShardInstanceState; +import com.inteligr8.alfresco.asie.model.ShardSet; +import com.inteligr8.alfresco.asie.model.SolrHost; +import com.inteligr8.alfresco.cachext.CollectionCache; +import com.inteligr8.alfresco.cachext.MultiValueCache; + +@Component +public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.ShardDiscoveryService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDSETS_CACHE) + private SimpleCache shardsetsCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODES_CACHE) + private SimpleCache nodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARD_NODES_CACHE) + private MultiValueCache shardNodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_STATE_CACHE) + private SimpleCache shardInstanceStatesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_UNAVAIL_CACHE) + private CollectionCache> nodeUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_DISABLE_CACHE) + private CollectionCache> nodeDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_UNAVAIL_CACHE) + private CollectionCache> shardInstanceUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_DISABLE_CACHE) + private CollectionCache> shardInstanceDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) + private SimpleCache coreExplicitIdCache; + + @Override + public ShardSet findSetByCore(String core) { + return this.shardsetsCache.get(core); + } + + @Override + public SolrHost findNode(String nodeHostname, int nodePort) { + Map resolvedAddresses = new HashMap<>(); + + for (String nodeSpec : this.nodesCache.getKeys()) { + SolrHost node = this.nodesCache.get(nodeSpec); + + if (!nodeHostname.equalsIgnoreCase(node.getHostname())) { + if (!resolvedAddresses.containsKey(nodeHostname)) + resolvedAddresses.put(nodeHostname, this.resolve(nodeHostname)); + InetAddress nodeAddress = resolvedAddresses.get(nodeHostname); + this.logger.trace("Resolved: {} => {}", nodeHostname, nodeAddress); + if (nodeAddress == null) + continue; + + if (!resolvedAddresses.containsKey(node.getHostname())) + resolvedAddresses.put(node.getHostname(), this.resolve(node.getHostname())); + InetAddress shardInstanceAddress = resolvedAddresses.get(node.getHostname()); + this.logger.trace("Resolved: {} => {}", node.getHostname(), shardInstanceAddress); + if (!nodeAddress.equals(shardInstanceAddress)) + continue; + } + + if (nodePort == node.getPort()) { + this.logger.debug("Found node: {}", node); + return node; + } + } + + return null; + } + + private InetAddress resolve(String hostname) { + try { + return InetAddress.getByName(hostname); + } catch (UnknownHostException uhe) { + return null; + } + } + + @Override + public Map> findByNode(SolrHost node) { + Map> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(shard.extractShardSetCore()); + + if (this.shardNodesCache.contains(shard, node)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + + Map shards = response.get(shardSet); + if (shards == null) + response.put(shardSet, shards = new HashMap<>()); + shards.put(shard.extractShardId(), state); + } + } + + return response; + } + + @Override + public Set findSetsByShardMethod(ShardMethodEnum... shardMethods) { + Set shardSets = new HashSet<>(); + + Set methods = CollectionUtils.asSet(shardMethods); + for (String core : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(core); + if (methods.contains(shardSet.getMethod())) + shardSets.add(shardSet); + } + + return shardSets; + } + + @Override + public Set findNodes(ShardSet shardSet) { + Set nodes = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore())) + nodes.addAll(this.shardNodesCache.get(shard)); + } + + return nodes; + } + + @Override + public Set findNodesByShard(ShardSet shardSet, int shardId) { + Set nodes = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore()) && shardId == shard.extractShardId()) + nodes.addAll(this.shardNodesCache.get(shard)); + } + + return nodes; + } + + @Override + public Map> findLatestNodeStates(ShardSet shardSet) { + Map> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (!shardSet.getCore().equals(shard.extractShardSetCore())) + continue; + + SolrHost latestNode = null; + ShardInstanceState latestState = null; + + for (SolrHost node : this.shardNodesCache.get(shard)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + if (latestState == null || state.compareTo(latestState) < 0) { + latestState = state; + latestNode = node; + } + } + + if (latestNode != null) + response.put(shard.extractShardId(), new Pair<>(latestNode, latestState)); + } + + return response; + } + + @Override + public List> findNodeStatesByShard(ShardSet shardSet, int shardId) { + List> response = new LinkedList<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (!shardSet.getCore().equals(shard.extractShardSetCore())) + continue; + + for (SolrHost node : this.shardNodesCache.get(shard)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + response.add(new Pair<>(node, state)); + } + } + + return response; + } + + @Override + public Set findIdsByNode(ShardSet shardSet, SolrHost node) { + Set shardIds = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore()) && this.shardNodesCache.contains(shard, node)) + shardIds.add(shard.extractShardId()); + } + + return shardIds; + } + + @Override + public Map findStatesByNode(ShardSet shardSet, SolrHost node) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java index 52381ae..3d3b3c5 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java @@ -24,12 +24,14 @@ import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.search.SearchParameters; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; +import org.antlr.runtime.RecognitionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEvent; +import org.springframework.context.annotation.Primary; import org.springframework.extensions.surf.util.AbstractLifecycleBean; import org.springframework.stereotype.Component; @@ -50,6 +52,7 @@ import com.inteligr8.alfresco.cachext.CollectionCache; import com.inteligr8.alfresco.cachext.MultiValueCache; @Component +@Primary public class SolrShardRegistry extends AbstractLifecycleBean implements ShardRegistry { private final Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -108,8 +111,8 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) private SimpleCache coreExplicitIdCache; - @Value("${inteligr8.asie.registerUnknownShardOffline}") - private boolean registerOffline; + @Value("${inteligr8.asie.registerUnknownShardDisabled}") + private boolean registerDisabled; @Value("${inteligr8.asie.offlineIdleShardInSeconds}") private int offlineIdleShardInSeconds; @@ -212,7 +215,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg try { this.persistShardSetCache(); this.persistNodeCache(); - this.persistShardNodeCache(); + this.persistShardNodesCache(); this.persistShardInstanceCache(); } finally { this.jobLockService.releaseLock(lockId, this.shardLock); @@ -258,7 +261,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE); } - private void persistShardNodeCache() { + private void persistShardNodesCache() { // add anything missing // update anything changed for (Shard shard : this.shardNodesCache.getKeys()) { @@ -277,17 +280,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg // update anything changed for (ShardInstance shardNode : this.shardInstanceStatesCache.getKeys()) { ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); - ShardInstanceState currentState = (ShardInstanceState) this.attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - if (currentState != null) { - if (currentState.compareTo(state) >= 0) { - // do nothing - } else { - this.logger.debug("The persisted state was old; updating: {}: {} => {}", shardNode, currentState, state); - this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - } - } else { - this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - } + this.checkSetAttribute(state, shardNode); } // we are not removing anything removed from the cache, as it might have expired @@ -309,15 +302,18 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg }, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE); } - protected void persistCache(ShardSet shardSet, SolrHost node, Shard shard, ShardInstance shardNode, ShardInstanceState state) { - String lockId = this.jobLockService.getLock(this.shardLock, 2000L, 100L, 50); - try { - this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, shardSet.getCore()); - this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, node.getSpec()); - this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); - this.checkSetAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - } finally { - this.jobLockService.releaseLock(lockId, this.shardLock); + private void checkSetAttribute(ShardInstanceState state, ShardInstance shardNode) { + ShardInstanceState currentState = (ShardInstanceState) this.attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + if (currentState != null) { + if (currentState.compareTo(state) >= 0) { + // current state is older (greater; further down the list) + // do nothing + } else { + this.logger.debug("The persisted state was old; updating: {}: {} => {}", shardNode, currentState, state); + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } + } else { + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); } } @@ -340,7 +336,28 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg ShardInstance shardNode = ShardInstance.from(shard, node); ShardInstanceState state = ShardInstanceState.from(shardNodeState); - this.persistCache(shardSet, node, shard, shardNode, state); + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + if (!this.shardsetsCache.contains(shardSet.getCore())) + this.shardsetsCache.put(shardSet.getCore(), shardSet); + this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, shardSet.getCore()); + + if (!this.nodesCache.contains(node.getSpec())) + this.nodesCache.put(node.getSpec(), node); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, node.getSpec()); + if (!this.shardNodesCache.contains(shard, node)) + this.shardNodesCache.add(shard, node); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + + ShardInstanceState currentState = this.shardInstanceStatesCache.get(shardNode); + if (currentState == null || currentState.compareTo(state) > 0) + this.shardInstanceStatesCache.put(shardNode, state); + this.checkSetAttribute(state, shardNode); + if (this.registerDisabled && !this.shardInstanceDisableCache.contains(shardNode)) + this.shardInstanceDisableCache.add(shardNode); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } } @Override @@ -350,12 +367,17 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg SolrHost node = SolrHost.from(shardInstance); ShardInstance shardNode = ShardInstance.from(shard, node); - this.shardInstanceStatesCache.remove(shardNode); - this.shardInstanceDisableCache.remove(shardNode); - this.shardInstanceUnavailableCache.remove(shardNode); - this.nodeDisableCache.remove(node); - this.nodeUnavailableCache.remove(node); - this.attrService.removeAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.shardInstanceStatesCache.remove(shardNode); + this.shardInstanceDisableCache.remove(shardNode); + this.shardInstanceUnavailableCache.remove(shardNode); + this.nodeDisableCache.remove(node); + this.nodeUnavailableCache.remove(node); + this.attrService.removeAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } } @Override @@ -426,6 +448,8 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg this.shardInstanceDisableCache.clear(); this.shardInstanceUnavailableCache.clear(); this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARD_NODE); + + this.coreExplicitIdCache.clear(); } finally { this.jobLockService.releaseLock(lockId, this.shardLock); } @@ -497,52 +521,84 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg for (String shardSetSpec : this.shardsetsCache.getKeys()) { ShardSet shardSet = this.shardsetsCache.get(shardSetSpec); - List shards = new LinkedList<>(); - switch (shardSet.getMethod()) { - case EXPLICIT_ID: - String property = shardSet.getPrefixedProperty(); - QName propertyQName = QName.createQName(property, this.namespaceService); - DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); - - QueryInspector inspector = this.queryInspectorFactory.selectQueryInspector(searchParameters); - if (inspector == null) - continue; - - Set shardIds = new HashSet<>(); - List values = inspector.findRequiredPropertyValues(searchParameters.getQuery(), searchParameters.getDefaultOperator(), propertyQName, dtdef); - for (QueryValue value : values) { - if (value instanceof QuerySingleValue) { - @SuppressWarnings("unchecked") - Number num = ((QuerySingleValue) value).getValue(); - shardIds.add(num.intValue()); - } else if (value instanceof QueryRangeValue) { - @SuppressWarnings("unchecked") - QueryRangeValue num = (QueryRangeValue) value; - int start = num.getStart().intValue(); - if (!num.isIncludeStart()) - start++; - int end = num.getStart().intValue(); - if (!num.isIncludeEnd()) - end--; - for (int shardId = start; shardId <= end; shardId++) - shardIds.add(shardId); - } - } - - shards.addAll(this.getIndexSlice(shardSet, shardIds)); - default: - // make no determination - } + Set shardIds = this.getIndexSlice(searchParameters, shardSet); + if (shardIds == null) + continue; + + List shards = this.selectRandomNodes(shardSet, shardIds); if (!shards.isEmpty() && (bestShards == null || shards.size() < bestShards.size())) bestShards = shards; + if (bestShards != null && bestShards.size() == 1) + break; } return bestShards; } - protected List getIndexSlice(ShardSet shardSet, Collection shardIds) { + protected Set getIndexSlice(SearchParameters searchParameters, ShardSet shardSet) { + try { + switch (shardSet.getMethod()) { + case EXPLICIT_ID: + return this.getExplicitIdIndexSlice(searchParameters, shardSet); + default: + // no optimization available + return null; + } + } catch (RecognitionException re) { + this.logger.debug("Failed to parse the query: " + searchParameters.getQuery(), re); + // no optimization available + return null; + } + } + + protected Set getExplicitIdIndexSlice(SearchParameters searchParameters, ShardSet shardSet) throws RecognitionException { + this.logger.trace("Found {} shard set, which is the highest priority", shardSet.getMethod()); + + QueryInspector inspector = this.queryInspectorFactory.selectQueryInspector(searchParameters); + if (inspector == null) { + this.logger.debug("The search is using an unsupported query language; unable to optimize for {}: {}", shardSet.getMethod(), searchParameters.getLanguage()); + return null; + } + + String property = shardSet.getPrefixedProperty(); + QName propertyQName = QName.createQName(property, this.namespaceService); + this.logger.trace("Will attempt to see if search has a required constraint on explicit shard ID property: {}", propertyQName); + DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); + + Set shardIds = new HashSet<>(); + List values = inspector.findRequiredPropertyValues(searchParameters.getQuery(), searchParameters.getDefaultOperator(), propertyQName, dtdef); + this.logger.trace("Found {} matching terms query: {}: {}", values.size(), propertyQName, searchParameters.getQuery()); + for (QueryValue value : values) { + if (value instanceof QuerySingleValue) { + @SuppressWarnings("unchecked") + Number num = ((QuerySingleValue) value).getValue(); + shardIds.add(num.intValue()); + } else if (value instanceof QueryRangeValue) { + @SuppressWarnings("unchecked") + QueryRangeValue num = (QueryRangeValue) value; + int start = num.getStart().intValue(); + if (!num.isIncludeStart()) + start++; + int end = num.getStart().intValue(); + if (!num.isIncludeEnd()) + end--; + for (int shardId = start; shardId <= end; shardId++) + shardIds.add(shardId); + } + } + + if (shardIds.isEmpty()) { + this.logger.trace("The {} shard set cannot not be used to optimize the query", shardSet.getMethod()); + return null; + } + this.logger.debug("The {} shard set was used to optimize the query to use only shards: {}", shardSet.getMethod(), shardIds); + + return shardIds; + } + + protected List selectRandomNodes(ShardSet shardSet, Collection shardIds) { List shardNodes = new LinkedList<>(); for (Integer shardId : shardIds) { diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties index b3963b8..db2025b 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties @@ -1,40 +1,109 @@ -inteligr8.asie.registerUnknownShardOffline=false -inteligr8.asie.idleShardExpirationInSeconds=${} +inteligr8.asie.registerUnknownShardDisabled=false +inteligr8.asie.offlineIdleShardInSeconds=120 +inteligr8.asie.forgetOfflineShardInSeconds=86400 +# we don't want items expiring out of the following caches +# an evicition policy of NONE disables the maxItems limits + # Overrides of alfresco-repository.jar/alfresco/caches.properties -cache.shardStateSharedCache.tx.maxItems=16384 -cache.shardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.shardStateSharedCache.maxItems=16384 -cache.shardStateSharedCache.timeToLiveSeconds=1800 -cache.shardStateSharedCache.maxIdleSeconds=0 -cache.shardStateSharedCache.cluster.type=fully-distributed -cache.shardStateSharedCache.backup-count=1 -cache.shardStateSharedCache.eviction-policy=LRU -cache.shardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy -cache.shardStateSharedCache.readBackupData=false +cache.asieShardsetsSharedCache.tx.maxItems=65536 +cache.asieShardsetsSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardsetsSharedCache.maxItems=65536 +cache.asieShardsetsSharedCache.timeToLiveSeconds=0 +cache.asieShardsetsSharedCache.maxIdleSeconds=0 +cache.asieShardsetsSharedCache.cluster.type=fully-distributed +cache.asieShardsetsSharedCache.backup-count=1 +cache.asieShardsetsSharedCache.eviction-policy=NONE +cache.asieShardsetsSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardsetsSharedCache.readBackupData=false -# maxItems needs to be greater than total shards, including HA instances -cache.offlineShardStateSharedCache.tx.maxItems=16384 -cache.offlineShardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.offlineShardStateSharedCache.maxItems=16384 -cache.offlineShardStateSharedCache.timeToLiveSeconds=1800 -cache.offlineShardStateSharedCache.maxIdleSeconds=0 -cache.offlineShardStateSharedCache.cluster.type=fully-distributed -cache.offlineShardStateSharedCache.backup-count=1 -cache.offlineShardStateSharedCache.eviction-policy=LRU -cache.offlineShardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy -cache.offlineShardStateSharedCache.readBackupData=false +cache.asieNodesSharedCache.tx.maxItems=65536 +cache.asieNodesSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodesSharedCache.maxItems=65536 +cache.asieNodesSharedCache.timeToLiveSeconds=0 +cache.asieNodesSharedCache.maxIdleSeconds=0 +cache.asieNodesSharedCache.cluster.type=fully-distributed +cache.asieNodesSharedCache.backup-count=1 +cache.asieNodesSharedCache.eviction-policy=NONE +cache.asieNodesSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodesSharedCache.readBackupData=false -cache.coreExplicitIdSharedCache.tx.maxItems=16384 -cache.coreExplicitIdSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.coreExplicitIdSharedCache.maxItems=16384 -cache.coreExplicitIdSharedCache.timeToLiveSeconds=1800 -cache.coreExplicitIdSharedCache.maxIdleSeconds=0 -cache.coreExplicitIdSharedCache.cluster.type=fully-distributed -cache.coreExplicitIdSharedCache.backup-count=1 -cache.coreExplicitIdSharedCache.eviction-policy=LRU -cache.coreExplicitIdSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy -cache.coreExplicitIdSharedCache.readBackupData=false +cache.asieShardNodesSharedCache.tx.maxItems=65536 +cache.asieShardNodesSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardNodesSharedCache.maxItems=65536 +cache.asieShardNodesSharedCache.timeToLiveSeconds=0 +cache.asieShardNodesSharedCache.maxIdleSeconds=0 +cache.asieShardNodesSharedCache.cluster.type=fully-distributed +cache.asieShardNodesSharedCache.backup-count=1 +cache.asieShardNodesSharedCache.eviction-policy=NONE +cache.asieShardNodesSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardNodesSharedCache.readBackupData=false + +cache.asieShardInstanceStateSharedCache.tx.maxItems=65536 +cache.asieShardInstanceStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceStateSharedCache.maxItems=65536 +cache.asieShardInstanceStateSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceStateSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceStateSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceStateSharedCache.backup-count=1 +cache.asieShardInstanceStateSharedCache.eviction-policy=NONE +cache.asieShardInstanceStateSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceStateSharedCache.readBackupData=false + +cache.asieNodeDisabledSharedCache.tx.maxItems=65536 +cache.asieNodeDisabledSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodeDisabledSharedCache.maxItems=65536 +cache.asieNodeDisabledSharedCache.timeToLiveSeconds=0 +cache.asieNodeDisabledSharedCache.maxIdleSeconds=0 +cache.asieNodeDisabledSharedCache.cluster.type=fully-distributed +cache.asieNodeDisabledSharedCache.backup-count=1 +cache.asieNodeDisabledSharedCache.eviction-policy=NONE +cache.asieNodeDisabledSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodeDisabledSharedCache.readBackupData=false + +cache.asieNodeUnavailableSharedCache.tx.maxItems=65536 +cache.asieNodeUnavailableSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodeUnavailableSharedCache.maxItems=65536 +cache.asieNodeUnavailableSharedCache.timeToLiveSeconds=0 +cache.asieNodeUnavailableSharedCache.maxIdleSeconds=0 +cache.asieNodeUnavailableSharedCache.cluster.type=fully-distributed +cache.asieNodeUnavailableSharedCache.backup-count=1 +cache.asieNodeUnavailableSharedCache.eviction-policy=NONE +cache.asieNodeUnavailableSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodeUnavailableSharedCache.readBackupData=false + +cache.asieShardInstanceDisabledSharedCache.tx.maxItems=65536 +cache.asieShardInstanceDisabledSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceDisabledSharedCache.maxItems=65536 +cache.asieShardInstanceDisabledSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceDisabledSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceDisabledSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceDisabledSharedCache.backup-count=1 +cache.asieShardInstanceDisabledSharedCache.eviction-policy=NONE +cache.asieShardInstanceDisabledSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceDisabledSharedCache.readBackupData=false + +cache.asieShardInstanceUnavailableSharedCache.tx.maxItems=65536 +cache.asieShardInstanceUnavailableSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceUnavailableSharedCache.maxItems=65536 +cache.asieShardInstanceUnavailableSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceUnavailableSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceUnavailableSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceUnavailableSharedCache.backup-count=1 +cache.asieShardInstanceUnavailableSharedCache.eviction-policy=NONE +cache.asieShardInstanceUnavailableSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceUnavailableSharedCache.readBackupData=false + +cache.asieCoreExplicitSharedCache.tx.maxItems=65536 +cache.asieCoreExplicitSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieCoreExplicitSharedCache.maxItems=65536 +cache.asieCoreExplicitSharedCache.timeToLiveSeconds=0 +cache.asieCoreExplicitSharedCache.maxIdleSeconds=0 +cache.asieCoreExplicitSharedCache.cluster.type=fully-distributed +cache.asieCoreExplicitSharedCache.backup-count=1 +cache.asieCoreExplicitSharedCache.eviction-policy=NONE +cache.asieCoreExplicitSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieCoreExplicitSharedCache.readBackupData=false diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml index 0eebd98..444172a 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml @@ -7,12 +7,44 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd"> - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties index 2d638d2..6c5ec20 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties @@ -7,4 +7,5 @@ module.version=${module.version} module.repo.version.min=23.0 # this is creating all sorts of problems; probably because of the non-standard versioning +module.depends.com.inteligr8.alfresco.cachext-platform-module=* module.depends.com.inteligr8.alfresco.cxf-jaxrs-platform-module=* diff --git a/enterprise-module/metadata.keystore b/enterprise-module/metadata.keystore new file mode 100644 index 0000000000000000000000000000000000000000..2c2a1d961273b300b1852d37d43e21676277feb5 GIT binary patch literal 409 zcmXqLVw}pv$ZXKWXwSx})#lOmotKfFaX}NK0ZS934p3MFh*eOfM1fL*Kw(}W=49iB z>f+&IWL?m>*`RR)vJlgP#sw^ma}6428Kl8YVdXW5G_XK$c?`5!L=N{!9_e#B605cle)lJ&&JDO?$*;d>UWc8v;Y=9$(Y z?eMAldWXglLl*;ixMMg)429$b7;+g>8A=!u8B%~4NE;|36o?v%un2`@=B6qbnj0A# zm>QXw8(SC}8Tc9)8t}rM#>B|Vz@jjH<;y>ndk;UBxGpCCrcU6NQrG?ks@Yqco;K

dmVXfd literal 0 HcmV?d00001 diff --git a/enterprise-module/pom.xml b/enterprise-module/pom.xml index c072d81..b001787 100644 --- a/enterprise-module/pom.xml +++ b/enterprise-module/pom.xml @@ -16,8 +16,12 @@ ASIE Platform Module for ACS Enterprise - 5.2.0 + 4.9.0 23.3.0 + 23.3.0.98 + 10-2.1 + + true @@ -80,7 +84,7 @@ com.inteligr8.alfresco cxf-jaxrs-platform-module 1.3.1-acs-v23.3 - provided + amp diff --git a/enterprise-module/rad.sh b/enterprise-module/rad.sh index 7cb0a80..3c7c2fa 100644 --- a/enterprise-module/rad.sh +++ b/enterprise-module/rad.sh @@ -1,22 +1,22 @@ #!/bin/sh discoverArtifactId() { - ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate` + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'` } rebuild() { echo "Rebuilding project ..." - mvn process-classes + mvn process-test-classes } start() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad process-classes + mvn -Drad process-test-classes } start_log() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad -Ddocker.showLogs process-classes + mvn -Drad -Ddocker.showLogs process-test-classes } stop() { diff --git a/shared/pom.xml b/shared/pom.xml index e0e48dc..9dd66b6 100644 --- a/shared/pom.xml +++ b/shared/pom.xml @@ -16,7 +16,7 @@ ASIE Shared Library for Platform Modules - 5.2.0 + 4.9.0 23.3.0