diff --git a/community-module/metadata.keystore b/community-module/metadata.keystore new file mode 100644 index 0000000..2c2a1d9 Binary files /dev/null and b/community-module/metadata.keystore differ diff --git a/community-module/pom.xml b/community-module/pom.xml index 9ee7607..15bbafe 100644 --- a/community-module/pom.xml +++ b/community-module/pom.xml @@ -6,7 +6,7 @@ com.inteligr8.alfresco asie-platform-module-parent - 1.1-SNAPSHOT + 1.2-SNAPSHOT ../ @@ -16,8 +16,12 @@ ASIE Platform Module for ACS Community - 5.2.0 - 7.0.0 + 4.9.0 + 23.3.0 + 23.3.0.98 + 10-2.1 + + true @@ -42,7 +46,6 @@ com.inteligr8.alfresco asie-shared ${project.version} - provided @@ -51,6 +54,14 @@ alfresco-repository provided + + + + com.inteligr8.alfresco + cxf-jaxrs-platform-module + 1.3.1-acs-v23.3 + amp + @@ -74,6 +85,10 @@ true + + com.inteligr8.ootbee:beedk-acs-search-rad-tile:[1.1.6,2.0.0) + + com.inteligr8.ootbee:beedk-acs-platform-self-rad-tile:[1.1.6,2.0.0) com.inteligr8.ootbee:beedk-acs-platform-module-tile:[1.1.6,2.0.0) diff --git a/community-module/rad.sh b/community-module/rad.sh index 7cb0a80..3c7c2fa 100644 --- a/community-module/rad.sh +++ b/community-module/rad.sh @@ -1,22 +1,22 @@ #!/bin/sh discoverArtifactId() { - ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate` + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'` } rebuild() { echo "Rebuilding project ..." - mvn process-classes + mvn process-test-classes } start() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad process-classes + mvn -Drad process-test-classes } start_log() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad -Ddocker.showLogs process-classes + mvn -Drad -Ddocker.showLogs process-test-classes } stop() { diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java index 7b0486b..44ca0d7 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java @@ -1,10 +1,12 @@ package com.inteligr8.alfresco.asie.compute; +import java.util.List; import java.util.Set; import org.alfresco.repo.search.impl.parsers.CMISLexer; import org.alfresco.repo.search.impl.parsers.CMISParser; import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; import org.alfresco.service.cmr.search.SearchService; import org.alfresco.service.namespace.QName; import org.antlr.runtime.ANTLRStringStream; @@ -31,8 +33,9 @@ public class CmisQueryInspector implements QueryInspector { } @Override - public QueryValue findRequiredProperty(String cmisQuery, Operator defaultOperator, QName property) throws RecognitionException { - Tree tree = this.parseCmis(cmisQuery, defaultOperator); + public List findRequiredPropertyValues(String query, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException { + Tree tree = this.parseCmis(query, defaultOperator); + throw new UnsupportedOperationException(); } protected Tree parseCmis(String cmisQuery, Operator defaultOperator) throws RecognitionException { diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java index bad416d..2e7035c 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java @@ -1,5 +1,6 @@ package com.inteligr8.alfresco.asie.compute; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,7 +15,7 @@ public class QueryInspectorFactory implements InitializingBean { @Autowired private List inspectors; - private Map languageInspectorMap; + private Map languageInspectorMap = new HashMap<>(); @Override public void afterPropertiesSet() throws Exception { diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java new file mode 100644 index 0000000..5cbbaf1 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java @@ -0,0 +1,240 @@ +package com.inteligr8.alfresco.asie.service; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.alfresco.repo.cache.SimpleCache; +import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.service.namespace.QName; +import org.alfresco.util.Pair; +import org.alfresco.util.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.CommunityConstants; +import com.inteligr8.alfresco.asie.model.Shard; +import com.inteligr8.alfresco.asie.model.ShardInstance; +import com.inteligr8.alfresco.asie.model.ShardInstanceState; +import com.inteligr8.alfresco.asie.model.ShardSet; +import com.inteligr8.alfresco.asie.model.SolrHost; +import com.inteligr8.alfresco.cachext.CollectionCache; +import com.inteligr8.alfresco.cachext.MultiValueCache; + +@Component +public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.ShardDiscoveryService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDSETS_CACHE) + private SimpleCache shardsetsCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODES_CACHE) + private SimpleCache nodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARD_NODES_CACHE) + private MultiValueCache shardNodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_STATE_CACHE) + private SimpleCache shardInstanceStatesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_UNAVAIL_CACHE) + private CollectionCache> nodeUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_DISABLE_CACHE) + private CollectionCache> nodeDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_UNAVAIL_CACHE) + private CollectionCache> shardInstanceUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_DISABLE_CACHE) + private CollectionCache> shardInstanceDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) + private SimpleCache coreExplicitIdCache; + + @Override + public ShardSet findSetByCore(String core) { + return this.shardsetsCache.get(core); + } + + @Override + public SolrHost findNode(String nodeHostname, int nodePort) { + Map resolvedAddresses = new HashMap<>(); + + for (String nodeSpec : this.nodesCache.getKeys()) { + SolrHost node = this.nodesCache.get(nodeSpec); + + if (!nodeHostname.equalsIgnoreCase(node.getHostname())) { + if (!resolvedAddresses.containsKey(nodeHostname)) + resolvedAddresses.put(nodeHostname, this.resolve(nodeHostname)); + InetAddress nodeAddress = resolvedAddresses.get(nodeHostname); + this.logger.trace("Resolved: {} => {}", nodeHostname, nodeAddress); + if (nodeAddress == null) + continue; + + if (!resolvedAddresses.containsKey(node.getHostname())) + resolvedAddresses.put(node.getHostname(), this.resolve(node.getHostname())); + InetAddress shardInstanceAddress = resolvedAddresses.get(node.getHostname()); + this.logger.trace("Resolved: {} => {}", node.getHostname(), shardInstanceAddress); + if (!nodeAddress.equals(shardInstanceAddress)) + continue; + } + + if (nodePort == node.getPort()) { + this.logger.debug("Found node: {}", node); + return node; + } + } + + return null; + } + + private InetAddress resolve(String hostname) { + try { + return InetAddress.getByName(hostname); + } catch (UnknownHostException uhe) { + return null; + } + } + + @Override + public Map> findByNode(SolrHost node) { + Map> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(shard.extractShardSetCore()); + + if (this.shardNodesCache.contains(shard, node)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + + Map shards = response.get(shardSet); + if (shards == null) + response.put(shardSet, shards = new HashMap<>()); + shards.put(shard.extractShardId(), state); + } + } + + return response; + } + + @Override + public Set findSetsByShardMethod(ShardMethodEnum... shardMethods) { + Set shardSets = new HashSet<>(); + + Set methods = CollectionUtils.asSet(shardMethods); + for (String core : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(core); + if (methods.contains(shardSet.getMethod())) + shardSets.add(shardSet); + } + + return shardSets; + } + + @Override + public Set findNodes(ShardSet shardSet) { + Set nodes = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore())) + nodes.addAll(this.shardNodesCache.get(shard)); + } + + return nodes; + } + + @Override + public Set findNodesByShard(ShardSet shardSet, int shardId) { + Set nodes = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore()) && shardId == shard.extractShardId()) + nodes.addAll(this.shardNodesCache.get(shard)); + } + + return nodes; + } + + @Override + public Map> findLatestNodeStates(ShardSet shardSet) { + Map> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (!shardSet.getCore().equals(shard.extractShardSetCore())) + continue; + + SolrHost latestNode = null; + ShardInstanceState latestState = null; + + for (SolrHost node : this.shardNodesCache.get(shard)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + if (latestState == null || state.compareTo(latestState) < 0) { + latestState = state; + latestNode = node; + } + } + + if (latestNode != null) + response.put(shard.extractShardId(), new Pair<>(latestNode, latestState)); + } + + return response; + } + + @Override + public List> findNodeStatesByShard(ShardSet shardSet, int shardId) { + List> response = new LinkedList<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (!shardSet.getCore().equals(shard.extractShardSetCore())) + continue; + + for (SolrHost node : this.shardNodesCache.get(shard)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + response.add(new Pair<>(node, state)); + } + } + + return response; + } + + @Override + public Set findIdsByNode(ShardSet shardSet, SolrHost node) { + Set shardIds = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore()) && this.shardNodesCache.contains(shard, node)) + shardIds.add(shard.extractShardId()); + } + + return shardIds; + } + + @Override + public Map findStatesByNode(ShardSet shardSet, SolrHost node) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java index 52381ae..3d3b3c5 100644 --- a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java @@ -24,12 +24,14 @@ import org.alfresco.service.cmr.dictionary.DictionaryService; import org.alfresco.service.cmr.search.SearchParameters; import org.alfresco.service.namespace.NamespaceService; import org.alfresco.service.namespace.QName; +import org.antlr.runtime.RecognitionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEvent; +import org.springframework.context.annotation.Primary; import org.springframework.extensions.surf.util.AbstractLifecycleBean; import org.springframework.stereotype.Component; @@ -50,6 +52,7 @@ import com.inteligr8.alfresco.cachext.CollectionCache; import com.inteligr8.alfresco.cachext.MultiValueCache; @Component +@Primary public class SolrShardRegistry extends AbstractLifecycleBean implements ShardRegistry { private final Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -108,8 +111,8 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) private SimpleCache coreExplicitIdCache; - @Value("${inteligr8.asie.registerUnknownShardOffline}") - private boolean registerOffline; + @Value("${inteligr8.asie.registerUnknownShardDisabled}") + private boolean registerDisabled; @Value("${inteligr8.asie.offlineIdleShardInSeconds}") private int offlineIdleShardInSeconds; @@ -212,7 +215,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg try { this.persistShardSetCache(); this.persistNodeCache(); - this.persistShardNodeCache(); + this.persistShardNodesCache(); this.persistShardInstanceCache(); } finally { this.jobLockService.releaseLock(lockId, this.shardLock); @@ -258,7 +261,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE); } - private void persistShardNodeCache() { + private void persistShardNodesCache() { // add anything missing // update anything changed for (Shard shard : this.shardNodesCache.getKeys()) { @@ -277,17 +280,7 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg // update anything changed for (ShardInstance shardNode : this.shardInstanceStatesCache.getKeys()) { ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); - ShardInstanceState currentState = (ShardInstanceState) this.attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - if (currentState != null) { - if (currentState.compareTo(state) >= 0) { - // do nothing - } else { - this.logger.debug("The persisted state was old; updating: {}: {} => {}", shardNode, currentState, state); - this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - } - } else { - this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - } + this.checkSetAttribute(state, shardNode); } // we are not removing anything removed from the cache, as it might have expired @@ -309,15 +302,18 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg }, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE); } - protected void persistCache(ShardSet shardSet, SolrHost node, Shard shard, ShardInstance shardNode, ShardInstanceState state) { - String lockId = this.jobLockService.getLock(this.shardLock, 2000L, 100L, 50); - try { - this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, shardSet.getCore()); - this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, node.getSpec()); - this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); - this.checkSetAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); - } finally { - this.jobLockService.releaseLock(lockId, this.shardLock); + private void checkSetAttribute(ShardInstanceState state, ShardInstance shardNode) { + ShardInstanceState currentState = (ShardInstanceState) this.attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + if (currentState != null) { + if (currentState.compareTo(state) >= 0) { + // current state is older (greater; further down the list) + // do nothing + } else { + this.logger.debug("The persisted state was old; updating: {}: {} => {}", shardNode, currentState, state); + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } + } else { + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); } } @@ -340,7 +336,28 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg ShardInstance shardNode = ShardInstance.from(shard, node); ShardInstanceState state = ShardInstanceState.from(shardNodeState); - this.persistCache(shardSet, node, shard, shardNode, state); + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + if (!this.shardsetsCache.contains(shardSet.getCore())) + this.shardsetsCache.put(shardSet.getCore(), shardSet); + this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, shardSet.getCore()); + + if (!this.nodesCache.contains(node.getSpec())) + this.nodesCache.put(node.getSpec(), node); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, node.getSpec()); + if (!this.shardNodesCache.contains(shard, node)) + this.shardNodesCache.add(shard, node); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + + ShardInstanceState currentState = this.shardInstanceStatesCache.get(shardNode); + if (currentState == null || currentState.compareTo(state) > 0) + this.shardInstanceStatesCache.put(shardNode, state); + this.checkSetAttribute(state, shardNode); + if (this.registerDisabled && !this.shardInstanceDisableCache.contains(shardNode)) + this.shardInstanceDisableCache.add(shardNode); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } } @Override @@ -350,12 +367,17 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg SolrHost node = SolrHost.from(shardInstance); ShardInstance shardNode = ShardInstance.from(shard, node); - this.shardInstanceStatesCache.remove(shardNode); - this.shardInstanceDisableCache.remove(shardNode); - this.shardInstanceUnavailableCache.remove(shardNode); - this.nodeDisableCache.remove(node); - this.nodeUnavailableCache.remove(node); - this.attrService.removeAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.shardInstanceStatesCache.remove(shardNode); + this.shardInstanceDisableCache.remove(shardNode); + this.shardInstanceUnavailableCache.remove(shardNode); + this.nodeDisableCache.remove(node); + this.nodeUnavailableCache.remove(node); + this.attrService.removeAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } } @Override @@ -426,6 +448,8 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg this.shardInstanceDisableCache.clear(); this.shardInstanceUnavailableCache.clear(); this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARD_NODE); + + this.coreExplicitIdCache.clear(); } finally { this.jobLockService.releaseLock(lockId, this.shardLock); } @@ -497,52 +521,84 @@ public class SolrShardRegistry extends AbstractLifecycleBean implements ShardReg for (String shardSetSpec : this.shardsetsCache.getKeys()) { ShardSet shardSet = this.shardsetsCache.get(shardSetSpec); - List shards = new LinkedList<>(); - switch (shardSet.getMethod()) { - case EXPLICIT_ID: - String property = shardSet.getPrefixedProperty(); - QName propertyQName = QName.createQName(property, this.namespaceService); - DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); - - QueryInspector inspector = this.queryInspectorFactory.selectQueryInspector(searchParameters); - if (inspector == null) - continue; - - Set shardIds = new HashSet<>(); - List values = inspector.findRequiredPropertyValues(searchParameters.getQuery(), searchParameters.getDefaultOperator(), propertyQName, dtdef); - for (QueryValue value : values) { - if (value instanceof QuerySingleValue) { - @SuppressWarnings("unchecked") - Number num = ((QuerySingleValue) value).getValue(); - shardIds.add(num.intValue()); - } else if (value instanceof QueryRangeValue) { - @SuppressWarnings("unchecked") - QueryRangeValue num = (QueryRangeValue) value; - int start = num.getStart().intValue(); - if (!num.isIncludeStart()) - start++; - int end = num.getStart().intValue(); - if (!num.isIncludeEnd()) - end--; - for (int shardId = start; shardId <= end; shardId++) - shardIds.add(shardId); - } - } - - shards.addAll(this.getIndexSlice(shardSet, shardIds)); - default: - // make no determination - } + Set shardIds = this.getIndexSlice(searchParameters, shardSet); + if (shardIds == null) + continue; + + List shards = this.selectRandomNodes(shardSet, shardIds); if (!shards.isEmpty() && (bestShards == null || shards.size() < bestShards.size())) bestShards = shards; + if (bestShards != null && bestShards.size() == 1) + break; } return bestShards; } - protected List getIndexSlice(ShardSet shardSet, Collection shardIds) { + protected Set getIndexSlice(SearchParameters searchParameters, ShardSet shardSet) { + try { + switch (shardSet.getMethod()) { + case EXPLICIT_ID: + return this.getExplicitIdIndexSlice(searchParameters, shardSet); + default: + // no optimization available + return null; + } + } catch (RecognitionException re) { + this.logger.debug("Failed to parse the query: " + searchParameters.getQuery(), re); + // no optimization available + return null; + } + } + + protected Set getExplicitIdIndexSlice(SearchParameters searchParameters, ShardSet shardSet) throws RecognitionException { + this.logger.trace("Found {} shard set, which is the highest priority", shardSet.getMethod()); + + QueryInspector inspector = this.queryInspectorFactory.selectQueryInspector(searchParameters); + if (inspector == null) { + this.logger.debug("The search is using an unsupported query language; unable to optimize for {}: {}", shardSet.getMethod(), searchParameters.getLanguage()); + return null; + } + + String property = shardSet.getPrefixedProperty(); + QName propertyQName = QName.createQName(property, this.namespaceService); + this.logger.trace("Will attempt to see if search has a required constraint on explicit shard ID property: {}", propertyQName); + DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); + + Set shardIds = new HashSet<>(); + List values = inspector.findRequiredPropertyValues(searchParameters.getQuery(), searchParameters.getDefaultOperator(), propertyQName, dtdef); + this.logger.trace("Found {} matching terms query: {}: {}", values.size(), propertyQName, searchParameters.getQuery()); + for (QueryValue value : values) { + if (value instanceof QuerySingleValue) { + @SuppressWarnings("unchecked") + Number num = ((QuerySingleValue) value).getValue(); + shardIds.add(num.intValue()); + } else if (value instanceof QueryRangeValue) { + @SuppressWarnings("unchecked") + QueryRangeValue num = (QueryRangeValue) value; + int start = num.getStart().intValue(); + if (!num.isIncludeStart()) + start++; + int end = num.getStart().intValue(); + if (!num.isIncludeEnd()) + end--; + for (int shardId = start; shardId <= end; shardId++) + shardIds.add(shardId); + } + } + + if (shardIds.isEmpty()) { + this.logger.trace("The {} shard set cannot not be used to optimize the query", shardSet.getMethod()); + return null; + } + this.logger.debug("The {} shard set was used to optimize the query to use only shards: {}", shardSet.getMethod(), shardIds); + + return shardIds; + } + + protected List selectRandomNodes(ShardSet shardSet, Collection shardIds) { List shardNodes = new LinkedList<>(); for (Integer shardId : shardIds) { diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties index b3963b8..db2025b 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties @@ -1,40 +1,109 @@ -inteligr8.asie.registerUnknownShardOffline=false -inteligr8.asie.idleShardExpirationInSeconds=${} +inteligr8.asie.registerUnknownShardDisabled=false +inteligr8.asie.offlineIdleShardInSeconds=120 +inteligr8.asie.forgetOfflineShardInSeconds=86400 +# we don't want items expiring out of the following caches +# an evicition policy of NONE disables the maxItems limits + # Overrides of alfresco-repository.jar/alfresco/caches.properties -cache.shardStateSharedCache.tx.maxItems=16384 -cache.shardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.shardStateSharedCache.maxItems=16384 -cache.shardStateSharedCache.timeToLiveSeconds=1800 -cache.shardStateSharedCache.maxIdleSeconds=0 -cache.shardStateSharedCache.cluster.type=fully-distributed -cache.shardStateSharedCache.backup-count=1 -cache.shardStateSharedCache.eviction-policy=LRU -cache.shardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy -cache.shardStateSharedCache.readBackupData=false +cache.asieShardsetsSharedCache.tx.maxItems=65536 +cache.asieShardsetsSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardsetsSharedCache.maxItems=65536 +cache.asieShardsetsSharedCache.timeToLiveSeconds=0 +cache.asieShardsetsSharedCache.maxIdleSeconds=0 +cache.asieShardsetsSharedCache.cluster.type=fully-distributed +cache.asieShardsetsSharedCache.backup-count=1 +cache.asieShardsetsSharedCache.eviction-policy=NONE +cache.asieShardsetsSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardsetsSharedCache.readBackupData=false -# maxItems needs to be greater than total shards, including HA instances -cache.offlineShardStateSharedCache.tx.maxItems=16384 -cache.offlineShardStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.offlineShardStateSharedCache.maxItems=16384 -cache.offlineShardStateSharedCache.timeToLiveSeconds=1800 -cache.offlineShardStateSharedCache.maxIdleSeconds=0 -cache.offlineShardStateSharedCache.cluster.type=fully-distributed -cache.offlineShardStateSharedCache.backup-count=1 -cache.offlineShardStateSharedCache.eviction-policy=LRU -cache.offlineShardStateSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy -cache.offlineShardStateSharedCache.readBackupData=false +cache.asieNodesSharedCache.tx.maxItems=65536 +cache.asieNodesSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodesSharedCache.maxItems=65536 +cache.asieNodesSharedCache.timeToLiveSeconds=0 +cache.asieNodesSharedCache.maxIdleSeconds=0 +cache.asieNodesSharedCache.cluster.type=fully-distributed +cache.asieNodesSharedCache.backup-count=1 +cache.asieNodesSharedCache.eviction-policy=NONE +cache.asieNodesSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodesSharedCache.readBackupData=false -cache.coreExplicitIdSharedCache.tx.maxItems=16384 -cache.coreExplicitIdSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} -cache.coreExplicitIdSharedCache.maxItems=16384 -cache.coreExplicitIdSharedCache.timeToLiveSeconds=1800 -cache.coreExplicitIdSharedCache.maxIdleSeconds=0 -cache.coreExplicitIdSharedCache.cluster.type=fully-distributed -cache.coreExplicitIdSharedCache.backup-count=1 -cache.coreExplicitIdSharedCache.eviction-policy=LRU -cache.coreExplicitIdSharedCache.merge-policy=com.hazelcast.spi.merge.PutIfAbsentMergePolicy -cache.coreExplicitIdSharedCache.readBackupData=false +cache.asieShardNodesSharedCache.tx.maxItems=65536 +cache.asieShardNodesSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardNodesSharedCache.maxItems=65536 +cache.asieShardNodesSharedCache.timeToLiveSeconds=0 +cache.asieShardNodesSharedCache.maxIdleSeconds=0 +cache.asieShardNodesSharedCache.cluster.type=fully-distributed +cache.asieShardNodesSharedCache.backup-count=1 +cache.asieShardNodesSharedCache.eviction-policy=NONE +cache.asieShardNodesSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardNodesSharedCache.readBackupData=false + +cache.asieShardInstanceStateSharedCache.tx.maxItems=65536 +cache.asieShardInstanceStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceStateSharedCache.maxItems=65536 +cache.asieShardInstanceStateSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceStateSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceStateSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceStateSharedCache.backup-count=1 +cache.asieShardInstanceStateSharedCache.eviction-policy=NONE +cache.asieShardInstanceStateSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceStateSharedCache.readBackupData=false + +cache.asieNodeDisabledSharedCache.tx.maxItems=65536 +cache.asieNodeDisabledSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodeDisabledSharedCache.maxItems=65536 +cache.asieNodeDisabledSharedCache.timeToLiveSeconds=0 +cache.asieNodeDisabledSharedCache.maxIdleSeconds=0 +cache.asieNodeDisabledSharedCache.cluster.type=fully-distributed +cache.asieNodeDisabledSharedCache.backup-count=1 +cache.asieNodeDisabledSharedCache.eviction-policy=NONE +cache.asieNodeDisabledSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodeDisabledSharedCache.readBackupData=false + +cache.asieNodeUnavailableSharedCache.tx.maxItems=65536 +cache.asieNodeUnavailableSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodeUnavailableSharedCache.maxItems=65536 +cache.asieNodeUnavailableSharedCache.timeToLiveSeconds=0 +cache.asieNodeUnavailableSharedCache.maxIdleSeconds=0 +cache.asieNodeUnavailableSharedCache.cluster.type=fully-distributed +cache.asieNodeUnavailableSharedCache.backup-count=1 +cache.asieNodeUnavailableSharedCache.eviction-policy=NONE +cache.asieNodeUnavailableSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodeUnavailableSharedCache.readBackupData=false + +cache.asieShardInstanceDisabledSharedCache.tx.maxItems=65536 +cache.asieShardInstanceDisabledSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceDisabledSharedCache.maxItems=65536 +cache.asieShardInstanceDisabledSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceDisabledSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceDisabledSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceDisabledSharedCache.backup-count=1 +cache.asieShardInstanceDisabledSharedCache.eviction-policy=NONE +cache.asieShardInstanceDisabledSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceDisabledSharedCache.readBackupData=false + +cache.asieShardInstanceUnavailableSharedCache.tx.maxItems=65536 +cache.asieShardInstanceUnavailableSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceUnavailableSharedCache.maxItems=65536 +cache.asieShardInstanceUnavailableSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceUnavailableSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceUnavailableSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceUnavailableSharedCache.backup-count=1 +cache.asieShardInstanceUnavailableSharedCache.eviction-policy=NONE +cache.asieShardInstanceUnavailableSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceUnavailableSharedCache.readBackupData=false + +cache.asieCoreExplicitSharedCache.tx.maxItems=65536 +cache.asieCoreExplicitSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieCoreExplicitSharedCache.maxItems=65536 +cache.asieCoreExplicitSharedCache.timeToLiveSeconds=0 +cache.asieCoreExplicitSharedCache.maxIdleSeconds=0 +cache.asieCoreExplicitSharedCache.cluster.type=fully-distributed +cache.asieCoreExplicitSharedCache.backup-count=1 +cache.asieCoreExplicitSharedCache.eviction-policy=NONE +cache.asieCoreExplicitSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieCoreExplicitSharedCache.readBackupData=false diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml index 0eebd98..444172a 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml @@ -7,12 +7,44 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd"> - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties index 2d638d2..6c5ec20 100644 --- a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties @@ -7,4 +7,5 @@ module.version=${module.version} module.repo.version.min=23.0 # this is creating all sorts of problems; probably because of the non-standard versioning +module.depends.com.inteligr8.alfresco.cachext-platform-module=* module.depends.com.inteligr8.alfresco.cxf-jaxrs-platform-module=* diff --git a/enterprise-module/metadata.keystore b/enterprise-module/metadata.keystore new file mode 100644 index 0000000..2c2a1d9 Binary files /dev/null and b/enterprise-module/metadata.keystore differ diff --git a/enterprise-module/pom.xml b/enterprise-module/pom.xml index c072d81..b001787 100644 --- a/enterprise-module/pom.xml +++ b/enterprise-module/pom.xml @@ -16,8 +16,12 @@ ASIE Platform Module for ACS Enterprise - 5.2.0 + 4.9.0 23.3.0 + 23.3.0.98 + 10-2.1 + + true @@ -80,7 +84,7 @@ com.inteligr8.alfresco cxf-jaxrs-platform-module 1.3.1-acs-v23.3 - provided + amp diff --git a/enterprise-module/rad.sh b/enterprise-module/rad.sh index 7cb0a80..3c7c2fa 100644 --- a/enterprise-module/rad.sh +++ b/enterprise-module/rad.sh @@ -1,22 +1,22 @@ #!/bin/sh discoverArtifactId() { - ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate` + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'` } rebuild() { echo "Rebuilding project ..." - mvn process-classes + mvn process-test-classes } start() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad process-classes + mvn -Drad process-test-classes } start_log() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad -Ddocker.showLogs process-classes + mvn -Drad -Ddocker.showLogs process-test-classes } stop() { diff --git a/shared/pom.xml b/shared/pom.xml index e0e48dc..9dd66b6 100644 --- a/shared/pom.xml +++ b/shared/pom.xml @@ -16,7 +16,7 @@ ASIE Shared Library for Platform Modules - 5.2.0 + 4.9.0 23.3.0