diff --git a/community-module/.gitignore b/community-module/.gitignore new file mode 100644 index 0000000..e59065e --- /dev/null +++ b/community-module/.gitignore @@ -0,0 +1,12 @@ +# Maven +target +pom.xml.versionsBackup + +# Eclipse +.project +.classpath +.settings +.vscode + +# IDEA +/.idea/ diff --git a/community-module/README.md b/community-module/README.md new file mode 100644 index 0000000..1aa10d1 --- /dev/null +++ b/community-module/README.md @@ -0,0 +1 @@ +# ASIE Platform Module Library diff --git a/community-module/metadata.keystore b/community-module/metadata.keystore new file mode 100644 index 0000000..2c2a1d9 Binary files /dev/null and b/community-module/metadata.keystore differ diff --git a/community-module/pom.xml b/community-module/pom.xml new file mode 100644 index 0000000..15bbafe --- /dev/null +++ b/community-module/pom.xml @@ -0,0 +1,106 @@ + + 4.0.0 + + + com.inteligr8.alfresco + asie-platform-module-parent + 1.2-SNAPSHOT + ../ + + + asie-community-platform-module + jar + + ASIE Platform Module for ACS Community + + + 4.9.0 + 23.3.0 + 23.3.0.98 + 10-2.1 + + true + + + + + + org.alfresco + acs-community-packaging + ${alfresco.platform.version} + pom + import + + + + + + + com.inteligr8.alfresco + cachext-platform-module + 1.0-SNAPSHOT + + + com.inteligr8.alfresco + asie-shared + ${project.version} + + + + + org.alfresco + alfresco-repository + provided + + + + + com.inteligr8.alfresco + cxf-jaxrs-platform-module + 1.3.1-acs-v23.3 + amp + + + + + junit + junit + test + + + org.mockito + mockito-core + test + + + + + + + io.repaint.maven + tiles-maven-plugin + 2.40 + true + + + + com.inteligr8.ootbee:beedk-acs-search-rad-tile:[1.1.6,2.0.0) + + com.inteligr8.ootbee:beedk-acs-platform-self-rad-tile:[1.1.6,2.0.0) + + com.inteligr8.ootbee:beedk-acs-platform-module-tile:[1.1.6,2.0.0) + + + + + + + + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + + diff --git a/community-module/rad.ps1 b/community-module/rad.ps1 new file mode 100644 index 0000000..61bcb2f --- /dev/null +++ b/community-module/rad.ps1 @@ -0,0 +1,74 @@ + +function discoverArtifactId { + $script:ARTIFACT_ID=(mvn -q -Dexpression=project"."artifactId -DforceStdout help:evaluate) +} + +function rebuild { + echo "Rebuilding project ..." + mvn process-classes +} + +function start_ { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad process-classes +} + +function start_log { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad "-Ddocker.showLogs" process-classes +} + +function stop_ { + discoverArtifactId + echo "Stopping Docker containers that supported rapid application development ..." + docker container ls --filter name=${ARTIFACT_ID}-* + echo "Stopping containers ..." + docker container stop (docker container ls -q --filter name=${ARTIFACT_ID}-*) + echo "Removing containers ..." + docker container rm (docker container ls -aq --filter name=${ARTIFACT_ID}-*) +} + +function tail_logs { + param ( + $container + ) + + discoverArtifactId + docker container logs -f (docker container ls -q --filter name=${ARTIFACT_ID}-${container}) +} + +function list { + discoverArtifactId + docker container ls --filter name=${ARTIFACT_ID}-* +} + +switch ($args[0]) { + "start" { + start_ + } + "start_log" { + start_log + } + "stop" { + stop_ + } + "restart" { + stop_ + start_ + } + "rebuild" { + rebuild + } + "tail" { + tail_logs $args[1] + } + "containers" { + list + } + default { + echo "Usage: .\rad.ps1 [ start | start_log | stop | restart | rebuild | tail {container} | containers ]" + } +} + +echo "Completed!" + diff --git a/community-module/rad.sh b/community-module/rad.sh new file mode 100644 index 0000000..3c7c2fa --- /dev/null +++ b/community-module/rad.sh @@ -0,0 +1,71 @@ +#!/bin/sh + +discoverArtifactId() { + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'` +} + +rebuild() { + echo "Rebuilding project ..." + mvn process-test-classes +} + +start() { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad process-test-classes +} + +start_log() { + echo "Rebuilding project and starting Docker containers to support rapid application development ..." + mvn -Drad -Ddocker.showLogs process-test-classes +} + +stop() { + discoverArtifactId + echo "Stopping Docker containers that supported rapid application development ..." + docker container ls --filter name=${ARTIFACT_ID}-* + echo "Stopping containers ..." + docker container stop `docker container ls -q --filter name=${ARTIFACT_ID}-*` + echo "Removing containers ..." + docker container rm `docker container ls -aq --filter name=${ARTIFACT_ID}-*` +} + +tail_logs() { + discoverArtifactId + docker container logs -f `docker container ls -q --filter name=${ARTIFACT_ID}-$1` +} + +list() { + discoverArtifactId + docker container ls --filter name=${ARTIFACT_ID}-* +} + +case "$1" in + start) + start + ;; + start_log) + start_log + ;; + stop) + stop + ;; + restart) + stop + start + ;; + rebuild) + rebuild + ;; + tail) + tail_logs $2 + ;; + containers) + list + ;; + *) + echo "Usage: ./rad.sh [ start | start_log | stop | restart | rebuild | tail {container} | containers ]" + exit 1 +esac + +echo "Completed!" + diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java new file mode 100644 index 0000000..ce16419 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/CommunityConstants.java @@ -0,0 +1,23 @@ +package com.inteligr8.alfresco.asie; + +public interface CommunityConstants extends Constants { + + static final String BEAN_SHARDSETS_CACHE = "asieShardsetsCache"; + static final String BEAN_NODES_CACHE = "asieNodesCache"; + static final String BEAN_SHARD_NODES_CACHE = "asieShardNodesCache"; + static final String BEAN_SHARDINST_STATE_CACHE = "asieShardInstanceStateCache"; + static final String BEAN_NODE_DISABLE_CACHE = "asieNodeDisabledCache"; + static final String BEAN_NODE_UNAVAIL_CACHE = "asieNodeUnavailableCache"; + static final String BEAN_SHARDINST_DISABLE_CACHE = "asieShardInstanceDisabledCache"; + static final String BEAN_SHARDINST_UNAVAIL_CACHE = "asieShardInstanceUnavailableCache"; + static final String BEAN_CORE_EXPLICIT_CACHE = "asieCoreExplicitCache"; + + static final String ATTR_ASIE_SHARDSET = "inteligr8.asie.shardSet"; + static final String ATTR_ASIE_NODE = "inteligr8.asie.node"; + static final String ATTR_ASIE_SHARD_NODES = "inteligr8.asie.shard.nodes"; + static final String ATTR_ASIE_SHARD_NODE = "inteligr8.asie.shard.node"; + static final String ATTR_OBJECT = "object"; + static final String ATTR_DISABLE = "disabled"; + static final String ATTR_NODES = "nodes"; + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java new file mode 100644 index 0000000..44ca0d7 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/CmisQueryInspector.java @@ -0,0 +1,50 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.util.List; +import java.util.Set; + +import org.alfresco.repo.search.impl.parsers.CMISLexer; +import org.alfresco.repo.search.impl.parsers.CMISParser; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; +import org.apache.commons.collections4.SetUtils; +import org.springframework.stereotype.Component; + +@Component +public class CmisQueryInspector implements QueryInspector { + + private Set supportedLanguages = SetUtils.unmodifiableSet( + SearchService.LANGUAGE_CMIS_ALFRESCO, + SearchService.LANGUAGE_CMIS_STRICT, + SearchService.LANGUAGE_INDEX_CMIS, + SearchService.LANGUAGE_SOLR_CMIS); + + @Override + public Set getSupportedLanguages() { + return this.supportedLanguages; + } + + @Override + public List findRequiredPropertyValues(String query, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException { + Tree tree = this.parseCmis(query, defaultOperator); + throw new UnsupportedOperationException(); + } + + protected Tree parseCmis(String cmisQuery, Operator defaultOperator) throws RecognitionException { + CharStream cs = new ANTLRStringStream(cmisQuery); + CMISLexer lexer = new CMISLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + CMISParser parser = new CMISParser(tokens); + CommonTree tree = (CommonTree) parser.query().getTree(); + return tree; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java new file mode 100644 index 0000000..cb972ba --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/FtsQueryInspector.java @@ -0,0 +1,290 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Period; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.alfresco.repo.search.impl.parsers.FTSLexer; +import org.alfresco.repo.search.impl.parsers.FTSParser; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.repository.AssociationRef; +import org.alfresco.service.cmr.repository.ChildAssociationRef; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.cmr.search.SearchService; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; +import org.apache.commons.collections4.SetUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class FtsQueryInspector implements QueryInspector { + + private final Logger logger = LoggerFactory.getLogger(FtsQueryInspector.class); + + private final Set supportedLanguages = SetUtils.unmodifiableSet( + SearchService.LANGUAGE_FTS_ALFRESCO, + SearchService.LANGUAGE_INDEX_FTS_ALFRESCO, + SearchService.LANGUAGE_SOLR_FTS_ALFRESCO, + SearchService.LANGUAGE_LUCENE); + + @Autowired + private NamespaceService namespaceService; + + @Override + public Set getSupportedLanguages() { + return this.supportedLanguages; + } + + @Override + public List findRequiredPropertyValues(String ftsQuery, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException { + Tree tree = this.parseFts(ftsQuery, defaultOperator); + tree = this.bypassSingleTermDisjunctions(tree); + if (tree == null) + return null; + + Collection trees = this.extractRequiredTerms(tree); + this.logger.trace("Found {} required terms in query: {}", trees.size(), ftsQuery); + this.filterPropertyTerms(trees, property); + this.logger.trace("Found {} required terms for property {} in query: {}", trees.size(), property, ftsQuery); + this.filterOutFuzzyTerms(trees); + this.logger.trace("Found {} required definitive terms for property {} in query: {}", trees.size(), property, ftsQuery); + + List values = new ArrayList<>(trees.size()); + for (Tree t : trees) + values.add(this.extractValue(t, dataTypeDef)); + return values; + } + + protected Tree parseFts(String ftsQuery, Operator defaultOperator) throws RecognitionException { + CharStream cs = new ANTLRStringStream(ftsQuery); + FTSLexer lexer = new FTSLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + FTSParser parser = new FTSParser(tokens); + parser.setDefaultFieldConjunction(defaultOperator.equals(Operator.AND)); + parser.setMode(defaultOperator.equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); + CommonTree tree = (CommonTree) parser.ftsQuery().getTree(); + return tree; + } + + protected Tree bypassSingleTermDisjunctions(Tree tree) { + while ("DISJUNCTION".equals(tree.getText()) && tree.getChildCount() == 1) + tree = tree.getChild(0); + if ("DISJUNCTION".equals(tree.getText())) + return null; + return tree; + } + + protected Collection extractRequiredTerms(Tree tree) { + while ("DISJUNCTION".equals(tree.getText()) && tree.getChildCount() == 1) + tree = tree.getChild(0); + + List terms = new LinkedList<>(); + + switch (tree.getText()) { + case "DISJUNCTION": + break; + case "CONJUNCTION": + for (int c = 0; c < tree.getChildCount(); c++) { + Collection subtrees = this.extractRequiredTerms(tree.getChild(c)); + if (subtrees == null || subtrees.isEmpty()) + continue; + terms.addAll(subtrees); + } + break; + case "DEFAULT": + terms.add(tree); + break; + default: + this.logger.warn("Unexpected/unsupported tree: {}", tree.getText()); + } + + return terms; + } + + protected Collection filterPropertyTerms(Collection trees, QName property) { + if (trees.isEmpty()) + return trees; + + Set prefixes = new HashSet<>(this.namespaceService.getPrefixes(property.getNamespaceURI())); + if (prefixes.isEmpty()) { + this.logger.warn("Unexpected/unsupported namespace: {}", property.getNamespaceURI()); + trees.clear(); + return trees; + } + + Iterator i = trees.iterator(); + + while (i.hasNext()) { + Tree tree = i.next(); + + if ("DEFAULT".equals(tree.getText())) + tree = tree.getChild(0); + + int skip = -1; + switch (tree.getText()) { + case "TERM": + case "PHRASE": + case "EXACT_TERM": + case "EXACT_PHRASE": + skip = 1; // skip the value child + break; + case "RANGE": + skip = 4; // skip the inclusive, start, end, inclusive children + break; + default: + } + + if (skip >= 0) { + Tree fieldRef = tree.getChild(skip); + if (!"FIELD_REF".equals(fieldRef.getText())) { + this.logger.warn("Unexpected/unsupported tree: {}", tree.getText()); + } else if (!fieldRef.getChild(0).getText().equals(property.getLocalName())) { + this.logger.trace("Found but ignoring property: {}", fieldRef.getChild(0).getText()); + } else { + Tree prefix = fieldRef.getChild(1); + if (!"PREFIX".equals(prefix.getText())) { + this.logger.warn("Unexpected/unsupported tree: {}", tree.getText()); + } else if (!prefixes.contains(prefix.getChild(0).getText())) { + this.logger.trace("Found but ignoring property: {}:{}", prefix.getChild(0).getText(), property.getLocalName()); + } else { + // this will skip the remove() + continue; + } + } + } + + i.remove(); + } + + return trees; + } + + protected Collection filterOutFuzzyTerms(Collection trees) { + if (trees.isEmpty()) + return trees; + + Iterator i = trees.iterator(); + + while (i.hasNext()) { + Tree tree = i.next(); + + if ("DEFAULT".equals(tree.getText())) + tree = tree.getChild(0); + + switch (tree.getText()) { + case "EXACT_TERM": + case "EXACT_PHRASE": + case "RANGE": + break; + default: + i.remove(); + } + } + + return trees; + } + + protected QueryValue extractValue(Tree tree, DataTypeDefinition dataTypeDef) { + if ("DEFAULT".equals(tree.getText())) + tree = tree.getChild(0); + + switch (tree.getText()) { + case "RANGE": + return this.extractRangeValue(tree, dataTypeDef); + default: + } + + String value = this.unquote(tree.getChild(0).getText()); + + switch (dataTypeDef.getName().getLocalName()) { + case "boolean": + return new QuerySingleValue(Boolean.parseBoolean(value)); + case "double": + return new QuerySingleValue(Double.parseDouble(value)); + case "float": + return new QuerySingleValue(Float.parseFloat(value)); + case "int": + return new QuerySingleValue(Integer.parseInt(value)); + case "long": + return new QuerySingleValue(Long.parseLong(value)); + case "date": + return new QuerySingleValue(this.evaluateAsDate(value)); + case "datetime": + return new QuerySingleValue(this.evaluateAsDateTime(value)); + case "period": + return new QuerySingleValue(Period.parse(value)); + case "qname": + return new QuerySingleValue(QName.createQName(value, this.namespaceService)); + case "noderef": + return new QuerySingleValue(new NodeRef(value)); + case "childassocref": + return new QuerySingleValue(new ChildAssociationRef(value)); + case "assocref": + return new QuerySingleValue(new AssociationRef(value)); + case "locale": + return new QuerySingleValue(new Locale(value)); + default: + return new QuerySingleValue(value); + } + } + + protected QueryRangeValue extractRangeValue(Tree tree, DataTypeDefinition dataTypeDef) { + boolean includeStart = "INCLUSIVE".equals(tree.getChild(0).getText()); + String start = this.unquote(tree.getChild(1).getText()); + String end = this.unquote(tree.getChild(2).getText()); + boolean includeEnd = "INCLUSIVE".equals(tree.getChild(3).getText()); + + switch (dataTypeDef.getName().getLocalName()) { + case "double": + return new QueryRangeValue(includeStart, Double.parseDouble(start), includeEnd, Double.parseDouble(end)); + case "float": + return new QueryRangeValue(includeStart, Float.parseFloat(start), includeEnd, Float.parseFloat(end)); + case "int": + return new QueryRangeValue(includeStart, Integer.parseInt(start), includeEnd, Integer.parseInt(end)); + case "long": + return new QueryRangeValue(includeStart, Long.parseLong(start), includeEnd, Long.parseLong(end)); + case "date": + return new QueryRangeValue(includeStart, this.evaluateAsDate(start), includeEnd, this.evaluateAsDate(end)); + case "datetime": + return new QueryRangeValue(includeStart, this.evaluateAsDateTime(start), includeEnd, this.evaluateAsDateTime(end)); + default: + throw new UnsupportedOperationException("The data type does not make sense for range evaluation: " + dataTypeDef.getName()); + } + } + + protected LocalDate evaluateAsDate(String str) { + if ("now".equalsIgnoreCase(str)) return LocalDate.now(); + else return LocalDate.parse(str); + } + + protected LocalDateTime evaluateAsDateTime(String str) { + if ("now".equalsIgnoreCase(str)) return LocalDateTime.now(); + else return LocalDateTime.parse(str); + } + + protected String unquote(String str) { + if (str.length() < 2) return str; + else if (str.charAt(0) == '\'' && str.charAt(str.length()-1) == '\'') return str.substring(1, str.length()-1); + else if (str.charAt(0) == '\"' && str.charAt(str.length()-1) == '\"') return str.substring(1, str.length()-1); + else return str; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java new file mode 100644 index 0000000..6a1348d --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspector.java @@ -0,0 +1,74 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.util.List; +import java.util.Set; + +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.RecognitionException; + +public interface QueryInspector { + + Set getSupportedLanguages(); + + List findRequiredPropertyValues(String query, Operator defaultOperator, QName property, DataTypeDefinition dataTypeDef) throws RecognitionException; + + + + public interface QueryValue { + + } + + public class QuerySingleValue implements QueryValue { + + private T value; + + public QuerySingleValue(T value) { + this.value = value; + } + + public T getValue() { + return value; + } + + @Override + public String toString() { + return this.value.toString(); + } + + } + + public class QueryRangeValue implements QueryValue { + + private boolean includeStart; + private T start; + private boolean includeEnd; + private T end; + + public QueryRangeValue(boolean includeStart, T start, boolean includeEnd, T end) { + this.includeStart = includeStart; + this.start = start; + this.includeEnd = includeEnd; + this.end = end; + } + + public boolean isIncludeStart() { + return includeStart; + } + + public boolean isIncludeEnd() { + return includeEnd; + } + + public T getStart() { + return start; + } + + public T getEnd() { + return end; + } + + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java new file mode 100644 index 0000000..2e7035c --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/compute/QueryInspectorFactory.java @@ -0,0 +1,32 @@ +package com.inteligr8.alfresco.asie.compute; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.alfresco.service.cmr.search.SearchParameters; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class QueryInspectorFactory implements InitializingBean { + + @Autowired + private List inspectors; + + private Map languageInspectorMap = new HashMap<>(); + + @Override + public void afterPropertiesSet() throws Exception { + for (QueryInspector inspector : this.inspectors) { + for (String language : inspector.getSupportedLanguages()) + this.languageInspectorMap.put(language, inspector); + } + } + + public QueryInspector selectQueryInspector(SearchParameters searchParams) { + return this.languageInspectorMap.get(searchParams.getLanguage()); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java new file mode 100644 index 0000000..5355d0f --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/provider/ShardRegistryProvider.java @@ -0,0 +1,28 @@ +package com.inteligr8.alfresco.asie.provider; + +import org.alfresco.repo.index.shard.ShardRegistry; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; + +import com.inteligr8.alfresco.asie.Constants; + +@Configuration +public class ShardRegistryProvider extends AbstractProvider { + + /** + * This allows for the selection of the primary or first ShardRegistry + * registered in the Spring BeanFactory. + * + * @return A ShardRegistry. + */ + @Bean(Constants.BEAN_SHARD_REGISTRY) + @Qualifier(Constants.QUALIFIER_ASIE) + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public ShardRegistry selectBean() { + return this.getPrimary(ShardRegistry.class); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java new file mode 100644 index 0000000..5cbbaf1 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardDiscoveryService.java @@ -0,0 +1,240 @@ +package com.inteligr8.alfresco.asie.service; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.alfresco.repo.cache.SimpleCache; +import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.service.namespace.QName; +import org.alfresco.util.Pair; +import org.alfresco.util.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.CommunityConstants; +import com.inteligr8.alfresco.asie.model.Shard; +import com.inteligr8.alfresco.asie.model.ShardInstance; +import com.inteligr8.alfresco.asie.model.ShardInstanceState; +import com.inteligr8.alfresco.asie.model.ShardSet; +import com.inteligr8.alfresco.asie.model.SolrHost; +import com.inteligr8.alfresco.cachext.CollectionCache; +import com.inteligr8.alfresco.cachext.MultiValueCache; + +@Component +public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.ShardDiscoveryService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDSETS_CACHE) + private SimpleCache shardsetsCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODES_CACHE) + private SimpleCache nodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARD_NODES_CACHE) + private MultiValueCache shardNodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_STATE_CACHE) + private SimpleCache shardInstanceStatesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_UNAVAIL_CACHE) + private CollectionCache> nodeUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_DISABLE_CACHE) + private CollectionCache> nodeDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_UNAVAIL_CACHE) + private CollectionCache> shardInstanceUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_DISABLE_CACHE) + private CollectionCache> shardInstanceDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) + private SimpleCache coreExplicitIdCache; + + @Override + public ShardSet findSetByCore(String core) { + return this.shardsetsCache.get(core); + } + + @Override + public SolrHost findNode(String nodeHostname, int nodePort) { + Map resolvedAddresses = new HashMap<>(); + + for (String nodeSpec : this.nodesCache.getKeys()) { + SolrHost node = this.nodesCache.get(nodeSpec); + + if (!nodeHostname.equalsIgnoreCase(node.getHostname())) { + if (!resolvedAddresses.containsKey(nodeHostname)) + resolvedAddresses.put(nodeHostname, this.resolve(nodeHostname)); + InetAddress nodeAddress = resolvedAddresses.get(nodeHostname); + this.logger.trace("Resolved: {} => {}", nodeHostname, nodeAddress); + if (nodeAddress == null) + continue; + + if (!resolvedAddresses.containsKey(node.getHostname())) + resolvedAddresses.put(node.getHostname(), this.resolve(node.getHostname())); + InetAddress shardInstanceAddress = resolvedAddresses.get(node.getHostname()); + this.logger.trace("Resolved: {} => {}", node.getHostname(), shardInstanceAddress); + if (!nodeAddress.equals(shardInstanceAddress)) + continue; + } + + if (nodePort == node.getPort()) { + this.logger.debug("Found node: {}", node); + return node; + } + } + + return null; + } + + private InetAddress resolve(String hostname) { + try { + return InetAddress.getByName(hostname); + } catch (UnknownHostException uhe) { + return null; + } + } + + @Override + public Map> findByNode(SolrHost node) { + Map> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(shard.extractShardSetCore()); + + if (this.shardNodesCache.contains(shard, node)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + + Map shards = response.get(shardSet); + if (shards == null) + response.put(shardSet, shards = new HashMap<>()); + shards.put(shard.extractShardId(), state); + } + } + + return response; + } + + @Override + public Set findSetsByShardMethod(ShardMethodEnum... shardMethods) { + Set shardSets = new HashSet<>(); + + Set methods = CollectionUtils.asSet(shardMethods); + for (String core : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(core); + if (methods.contains(shardSet.getMethod())) + shardSets.add(shardSet); + } + + return shardSets; + } + + @Override + public Set findNodes(ShardSet shardSet) { + Set nodes = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore())) + nodes.addAll(this.shardNodesCache.get(shard)); + } + + return nodes; + } + + @Override + public Set findNodesByShard(ShardSet shardSet, int shardId) { + Set nodes = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore()) && shardId == shard.extractShardId()) + nodes.addAll(this.shardNodesCache.get(shard)); + } + + return nodes; + } + + @Override + public Map> findLatestNodeStates(ShardSet shardSet) { + Map> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (!shardSet.getCore().equals(shard.extractShardSetCore())) + continue; + + SolrHost latestNode = null; + ShardInstanceState latestState = null; + + for (SolrHost node : this.shardNodesCache.get(shard)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + if (latestState == null || state.compareTo(latestState) < 0) { + latestState = state; + latestNode = node; + } + } + + if (latestNode != null) + response.put(shard.extractShardId(), new Pair<>(latestNode, latestState)); + } + + return response; + } + + @Override + public List> findNodeStatesByShard(ShardSet shardSet, int shardId) { + List> response = new LinkedList<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (!shardSet.getCore().equals(shard.extractShardSetCore())) + continue; + + for (SolrHost node : this.shardNodesCache.get(shard)) { + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + response.add(new Pair<>(node, state)); + } + } + + return response; + } + + @Override + public Set findIdsByNode(ShardSet shardSet, SolrHost node) { + Set shardIds = new HashSet<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore()) && this.shardNodesCache.contains(shard, node)) + shardIds.add(shard.extractShardId()); + } + + return shardIds; + } + + @Override + public Map findStatesByNode(ShardSet shardSet, SolrHost node) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java new file mode 100644 index 0000000..5a43bf5 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/ShardStateService.java @@ -0,0 +1,25 @@ +package com.inteligr8.alfresco.asie.service; + +import org.alfresco.service.cmr.attributes.AttributeService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.Constants; + +@Component +public class ShardStateService implements com.inteligr8.alfresco.asie.spi.ShardStateService { + + @Autowired + @Qualifier(Constants.QUALIFIER_ASIE) + private AttributeService attrService; + + @Autowired + private SolrShardRegistry shardRegistry; + + @Override + public void clear() { + this.shardRegistry.purge(); + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java new file mode 100644 index 0000000..3d3b3c5 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardRegistry.java @@ -0,0 +1,628 @@ +package com.inteligr8.alfresco.asie.service; + +import java.io.Serializable; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Random; +import java.util.Set; + +import org.alfresco.repo.cache.SimpleCache; +import org.alfresco.repo.index.shard.Floc; +import org.alfresco.repo.index.shard.ShardState; +import org.alfresco.repo.lock.JobLockService; +import org.alfresco.service.cmr.attributes.AttributeService; +import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback; +import org.alfresco.service.cmr.dictionary.DataTypeDefinition; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.search.SearchParameters; +import org.alfresco.service.namespace.NamespaceService; +import org.alfresco.service.namespace.QName; +import org.antlr.runtime.RecognitionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.annotation.Primary; +import org.springframework.extensions.surf.util.AbstractLifecycleBean; +import org.springframework.stereotype.Component; + +import com.inteligr8.alfresco.asie.CommunityConstants; +import com.inteligr8.alfresco.asie.Constants; +import com.inteligr8.alfresco.asie.compute.QueryInspector; +import com.inteligr8.alfresco.asie.compute.QueryInspector.QueryRangeValue; +import com.inteligr8.alfresco.asie.compute.QueryInspector.QuerySingleValue; +import com.inteligr8.alfresco.asie.compute.QueryInspector.QueryValue; +import com.inteligr8.alfresco.asie.compute.QueryInspectorFactory; +import com.inteligr8.alfresco.asie.model.Shard; +import com.inteligr8.alfresco.asie.model.ShardInstance; +import com.inteligr8.alfresco.asie.model.ShardInstanceState; +import com.inteligr8.alfresco.asie.model.ShardSet; +import com.inteligr8.alfresco.asie.model.SolrHost; +import com.inteligr8.alfresco.asie.spi.ShardRegistry; +import com.inteligr8.alfresco.cachext.CollectionCache; +import com.inteligr8.alfresco.cachext.MultiValueCache; + +@Component +@Primary +public class SolrShardRegistry extends AbstractLifecycleBean implements ShardRegistry { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final Random random = new Random(); + private final QName shardLock = QName.createQName(Constants.NAMESPACE_ASIE, "shardLock"); + + @Autowired + @Qualifier(Constants.QUALIFIER_ASIE) + private AttributeService attrService; + + @Autowired + private NamespaceService namespaceService; + + @Autowired + private DictionaryService dictionaryService; + + @Autowired + private QueryInspectorFactory queryInspectorFactory; + + @Autowired + private JobLockService jobLockService; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDSETS_CACHE) + private SimpleCache shardsetsCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODES_CACHE) + private SimpleCache nodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARD_NODES_CACHE) + private MultiValueCache shardNodesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_STATE_CACHE) + private SimpleCache shardInstanceStatesCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_UNAVAIL_CACHE) + private CollectionCache> nodeUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_NODE_DISABLE_CACHE) + private CollectionCache> nodeDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_UNAVAIL_CACHE) + private CollectionCache> shardInstanceUnavailableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_SHARDINST_DISABLE_CACHE) + private CollectionCache> shardInstanceDisableCache; + + @Autowired + @Qualifier(CommunityConstants.BEAN_CORE_EXPLICIT_CACHE) + private SimpleCache coreExplicitIdCache; + + @Value("${inteligr8.asie.registerUnknownShardDisabled}") + private boolean registerDisabled; + + @Value("${inteligr8.asie.offlineIdleShardInSeconds}") + private int offlineIdleShardInSeconds; + + @Value("${inteligr8.asie.forgetOfflineShardInSeconds}") + private int forgetOfflineShardInSeconds; + + @Override + protected void onBootstrap(ApplicationEvent event) { + this.loadPersistedToCache(); + } + + @Override + protected void onShutdown(ApplicationEvent event) { + } + + protected void loadPersistedToCache() { + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + String core = (String) keys[1]; + if (!shardsetsCache.contains(core)) { + ShardSet shardSet = (ShardSet) value; + shardsetsCache.put(core, shardSet); + + switch (shardSet.getMethod()) { + case EXPLICIT_ID: + cacheExplicitShard(shardSet, false); + break; + default: + } + } + return true; + } + }, CommunityConstants.ATTR_ASIE_SHARDSET); + + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + String nodeSpec = (String) keys[2]; + SolrHost node = (SolrHost) value; + if (!nodesCache.contains(nodeSpec)) + nodesCache.put(nodeSpec, node); + if (Boolean.TRUE.equals(attrService.getAttribute(CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE, nodeSpec))) { + if (!nodeDisableCache.contains(node)) + nodeDisableCache.add(node); + } else if (nodeDisableCache.contains(node)) { + nodeDisableCache.remove(node); + } + return true; + } + }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT); + + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + Shard shard = (Shard) keys[1]; + SolrHost node = (SolrHost) keys[2]; + if (!shardNodesCache.contains(shard, node)) + shardNodesCache.add(shard, node); + return true; + } + }, CommunityConstants.ATTR_ASIE_SHARD_NODES); + + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + ShardInstance shardNode = (ShardInstance) keys[2]; + ShardInstanceState state = (ShardInstanceState) value; + if (!shardInstanceStatesCache.contains(shardNode)) + shardInstanceStatesCache.put(shardNode, state); + if (Boolean.TRUE.equals(attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE, shardNode))) { + if (!shardInstanceDisableCache.contains(shardNode)) + shardInstanceDisableCache.add(shardNode); + } else if (shardInstanceDisableCache.contains(shardNode)) { + shardInstanceDisableCache.remove(shardNode); + } + return true; + } + }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + private void cacheExplicitShard(ShardSet shardSet, boolean overwrite) { + if (overwrite || !this.coreExplicitIdCache.contains(shardSet.getCore())) { + String property = shardSet.getPrefixedProperty(); + QName propertyQName = QName.createQName(property, namespaceService); + + this.logger.debug("Mapping core to explicit ID: {} => {}", shardSet.getCore(), propertyQName); + this.coreExplicitIdCache.put(shardSet.getCore(), propertyQName); + } + } + + protected void persistCache() { + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 100L, 50); + try { + this.persistShardSetCache(); + this.persistNodeCache(); + this.persistShardNodesCache(); + this.persistShardInstanceCache(); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + private void persistShardSetCache() { + // add anything missing + // update anything changed + for (String core : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(core); + this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, core); + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + } + + private void persistNodeCache() { + // add anything missing + // update anything changed + for (String nodeSpec : this.nodesCache.getKeys()) { + SolrHost node = this.nodesCache.get(nodeSpec); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, nodeSpec); + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + + // add anything disabled + for (SolrHost node : this.nodeDisableCache.values()) + this.checkSetAttribute(Boolean.TRUE, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE, node.getSpec()); + + // remove anything not disabled + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + SolrHost node = SolrHost.from((String) keys[2]); + if (!nodeDisableCache.contains(node)) + attrService.removeAttribute(keys); + return true; + } + }, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_DISABLE); + } + + private void persistShardNodesCache() { + // add anything missing + // update anything changed + for (Shard shard : this.shardNodesCache.getKeys()) { + Collection nodes = this.shardNodesCache.get(shard); + for (SolrHost node : nodes) { + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + } + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + } + + private void persistShardInstanceCache() { + // add anything missing + // update anything changed + for (ShardInstance shardNode : this.shardInstanceStatesCache.getKeys()) { + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + this.checkSetAttribute(state, shardNode); + } + + // we are not removing anything removed from the cache, as it might have expired + // it will just recache on the next load + + // add anything disabled + for (ShardInstance shardNode : this.shardInstanceDisableCache.values()) + this.checkSetAttribute(Boolean.TRUE, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE, shardNode); + + // remove anything not disabled + this.attrService.getAttributes(new AttributeQueryCallback() { + @Override + public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) { + ShardInstance shardNode = (ShardInstance) keys[2]; + if (!shardInstanceDisableCache.contains(shardNode)) + attrService.removeAttribute(keys); + return true; + } + }, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_DISABLE); + } + + private void checkSetAttribute(ShardInstanceState state, ShardInstance shardNode) { + ShardInstanceState currentState = (ShardInstanceState) this.attrService.getAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + if (currentState != null) { + if (currentState.compareTo(state) >= 0) { + // current state is older (greater; further down the list) + // do nothing + } else { + this.logger.debug("The persisted state was old; updating: {}: {} => {}", shardNode, currentState, state); + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } + } else { + this.attrService.setAttribute(state, CommunityConstants.ATTR_ASIE_SHARD_NODE, CommunityConstants.ATTR_OBJECT, shardNode); + } + } + + private void checkSetAttribute(Serializable value, Serializable... keys) { + Serializable currentValue = this.attrService.getAttribute(keys); + if (currentValue != null) { + if (currentValue.equals(value)) + return; + this.logger.warn("The attribute value unexpectedly changed: {}: {} => {}", keys, currentValue, value); + } + + this.attrService.setAttribute(value, keys); + } + + @Override + public void registerShardState(ShardState shardNodeState) { + ShardSet shardSet = ShardSet.from(shardNodeState.getShardInstance().getShard().getFloc(), shardNodeState); + Shard shard = Shard.from(shardSet, shardNodeState.getShardInstance().getShard().getInstance()); + SolrHost node = SolrHost.from(shardNodeState.getShardInstance()); + ShardInstance shardNode = ShardInstance.from(shard, node); + ShardInstanceState state = ShardInstanceState.from(shardNodeState); + + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + if (!this.shardsetsCache.contains(shardSet.getCore())) + this.shardsetsCache.put(shardSet.getCore(), shardSet); + this.checkSetAttribute(shardSet, CommunityConstants.ATTR_ASIE_SHARDSET, shardSet.getCore()); + + if (!this.nodesCache.contains(node.getSpec())) + this.nodesCache.put(node.getSpec(), node); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_NODE, CommunityConstants.ATTR_OBJECT, node.getSpec()); + if (!this.shardNodesCache.contains(shard, node)) + this.shardNodesCache.add(shard, node); + this.checkSetAttribute(node, CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + + ShardInstanceState currentState = this.shardInstanceStatesCache.get(shardNode); + if (currentState == null || currentState.compareTo(state) > 0) + this.shardInstanceStatesCache.put(shardNode, state); + this.checkSetAttribute(state, shardNode); + if (this.registerDisabled && !this.shardInstanceDisableCache.contains(shardNode)) + this.shardInstanceDisableCache.add(shardNode); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + @Override + public void unregisterShardInstance(org.alfresco.repo.index.shard.ShardInstance shardInstance) { + ShardSet shardSet = ShardSet.from(shardInstance.getShard().getFloc(), null); + Shard shard = Shard.from(shardSet, shardInstance.getShard().getInstance()); + SolrHost node = SolrHost.from(shardInstance); + ShardInstance shardNode = ShardInstance.from(shard, node); + + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 500L, 10); + try { + this.shardInstanceStatesCache.remove(shardNode); + this.shardInstanceDisableCache.remove(shardNode); + this.shardInstanceUnavailableCache.remove(shardNode); + this.nodeDisableCache.remove(node); + this.nodeUnavailableCache.remove(node); + this.attrService.removeAttribute(CommunityConstants.ATTR_ASIE_SHARD_NODES, shard, node.getSpec()); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + @Override + public Map>> getFlocs() { + Map flocs = new HashMap<>(); + Map>> response = new HashMap<>(); + + for (Shard shard : this.shardNodesCache.getKeys()) { + String core = shard.extractShardSetCore(); + ShardSet shardSet = this.shardsetsCache.get(core); + + Map> shards; + Floc floc = flocs.get(core); + if (floc != null) { + floc = shardSet.toAlfrescoModel(); + shards = new HashMap<>(); + } else { + shards = response.get(floc); + } + + org.alfresco.repo.index.shard.Shard shard_ = shard.toAlfrescoModel(floc); + Set states = shards.get(shard_); + if (states == null) + states = new HashSet<>(); + + for (SolrHost node : this.shardNodesCache.get(shard)) { + if (this.nodeDisableCache.contains(node) || this.nodeUnavailableCache.contains(node)) { + this.logger.debug("Excluding node as it is disabled or considered unavailable: {}", node); + continue; + } + + ShardInstance shardNode = ShardInstance.from(shard, node); + if (this.shardInstanceDisableCache.contains(shardNode) || this.shardInstanceUnavailableCache.contains(shardNode)) { + this.logger.debug("Excluding shard node as it is disabled or considered unavailable: {}", shardNode); + continue; + } + + ShardInstanceState shardNodeState = this.shardInstanceStatesCache.get(shardNode); + states.add(shardNodeState.toAlfrescoModel(shardNode.toAlfrescoModel(shard_))); + } + + if (!states.isEmpty()) + shards.put(shard_, states); + if (!shards.isEmpty()) + response.put(floc, shards); + } + + return response; + } + + @Override + public void purge() { + String lockId = this.jobLockService.getLock(this.shardLock, 2500L, 100L, 50); + try { + this.logger.info("Removing all nodes/shards from the shard registry"); + this.shardsetsCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARDSET); + + this.nodesCache.clear(); + this.nodeDisableCache.clear(); + this.nodeUnavailableCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_NODE); + + this.shardNodesCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARD_NODES); + + this.shardInstanceStatesCache.clear(); + this.shardInstanceDisableCache.clear(); + this.shardInstanceUnavailableCache.clear(); + this.attrService.removeAttributes(CommunityConstants.ATTR_ASIE_SHARD_NODE); + + this.coreExplicitIdCache.clear(); + } finally { + this.jobLockService.releaseLock(lockId, this.shardLock); + } + } + + @Override + public void purgeAgedOutShards() { + OffsetDateTime onlineExpired = OffsetDateTime.now().minusSeconds(this.offlineIdleShardInSeconds); + OffsetDateTime offlineExpired = OffsetDateTime.now().minusSeconds(this.forgetOfflineShardInSeconds); + + for (ShardInstance shardNode : this.shardInstanceStatesCache.getKeys()) { + ShardInstanceState state = this.shardInstanceStatesCache.get(shardNode); + SolrHost node = shardNode.extractNode(); + + if (this.shardInstanceDisableCache.contains(shardNode)) { + this.logger.debug("Ignoring disabled shard instance during purgeAgedOutShards()"); + } else if (this.nodeDisableCache.contains(node)) { + this.logger.debug("Ignoring disabled node during purgeAgedOutShards()"); + } else if (state.getLastUpdated().isBefore(offlineExpired)) { + this.shardInstanceStatesCache.remove(shardNode); + if (this.shardInstanceUnavailableCache.remove(shardNode)) { + this.logger.info("Forgetting about already offline shard: {}", shardNode); + } else if (this.nodeUnavailableCache.remove(node)) { + this.logger.info("Forgetting about already offline shard: {}", shardNode); + } else { + this.logger.warn("Forgetting about online shard: {}", shardNode); + } + } else if (state.getLastUpdated().isBefore(onlineExpired)) { + this.logger.warn("Taking shard offline: {}", shardNode); + this.shardInstanceUnavailableCache.add(shardNode); + } + } + } + + @Override + public QName getExplicitIdProperty(String coreName) { + return this.coreExplicitIdCache.get(coreName); + } + + @Override + public Set getShardInstanceList(String coreName) { + Set shardIds = new HashSet<>(); + + ShardSet shardSet = this.shardsetsCache.get(coreName); + if (shardSet == null) + return Collections.emptySet(); + + + for (Shard shard : this.shardNodesCache.getKeys()) { + if (shardSet.getCore().equals(shard.extractShardSetCore())) { + shardIds.add(shard.extractShardId()); + } + } + + return shardIds; + } + + @Override + public OptionalInt getShardInstanceByTransactionTimestamp(String coreId, long txnTimestamp) { + throw new UnsupportedOperationException(); + } + + @Override + public List getIndexSlice(SearchParameters searchParameters) { + if (searchParameters.getQuery() == null) + return Collections.emptyList(); + + List bestShards = null; + + for (String shardSetSpec : this.shardsetsCache.getKeys()) { + ShardSet shardSet = this.shardsetsCache.get(shardSetSpec); + + Set shardIds = this.getIndexSlice(searchParameters, shardSet); + if (shardIds == null) + continue; + + List shards = this.selectRandomNodes(shardSet, shardIds); + + if (!shards.isEmpty() && (bestShards == null || shards.size() < bestShards.size())) + bestShards = shards; + if (bestShards != null && bestShards.size() == 1) + break; + } + + return bestShards; + } + + protected Set getIndexSlice(SearchParameters searchParameters, ShardSet shardSet) { + try { + switch (shardSet.getMethod()) { + case EXPLICIT_ID: + return this.getExplicitIdIndexSlice(searchParameters, shardSet); + default: + // no optimization available + return null; + } + } catch (RecognitionException re) { + this.logger.debug("Failed to parse the query: " + searchParameters.getQuery(), re); + // no optimization available + return null; + } + } + + protected Set getExplicitIdIndexSlice(SearchParameters searchParameters, ShardSet shardSet) throws RecognitionException { + this.logger.trace("Found {} shard set, which is the highest priority", shardSet.getMethod()); + + QueryInspector inspector = this.queryInspectorFactory.selectQueryInspector(searchParameters); + if (inspector == null) { + this.logger.debug("The search is using an unsupported query language; unable to optimize for {}: {}", shardSet.getMethod(), searchParameters.getLanguage()); + return null; + } + + String property = shardSet.getPrefixedProperty(); + QName propertyQName = QName.createQName(property, this.namespaceService); + this.logger.trace("Will attempt to see if search has a required constraint on explicit shard ID property: {}", propertyQName); + DataTypeDefinition dtdef = this.dictionaryService.getProperty(propertyQName).getDataType(); + + Set shardIds = new HashSet<>(); + List values = inspector.findRequiredPropertyValues(searchParameters.getQuery(), searchParameters.getDefaultOperator(), propertyQName, dtdef); + this.logger.trace("Found {} matching terms query: {}: {}", values.size(), propertyQName, searchParameters.getQuery()); + for (QueryValue value : values) { + if (value instanceof QuerySingleValue) { + @SuppressWarnings("unchecked") + Number num = ((QuerySingleValue) value).getValue(); + shardIds.add(num.intValue()); + } else if (value instanceof QueryRangeValue) { + @SuppressWarnings("unchecked") + QueryRangeValue num = (QueryRangeValue) value; + int start = num.getStart().intValue(); + if (!num.isIncludeStart()) + start++; + int end = num.getStart().intValue(); + if (!num.isIncludeEnd()) + end--; + for (int shardId = start; shardId <= end; shardId++) + shardIds.add(shardId); + } + } + + if (shardIds.isEmpty()) { + this.logger.trace("The {} shard set cannot not be used to optimize the query", shardSet.getMethod()); + return null; + } + this.logger.debug("The {} shard set was used to optimize the query to use only shards: {}", shardSet.getMethod(), shardIds); + + return shardIds; + } + + protected List selectRandomNodes(ShardSet shardSet, Collection shardIds) { + List shardNodes = new LinkedList<>(); + + for (Integer shardId : shardIds) { + Shard shard = Shard.from(shardSet, shardId); + + Collection nodes = this.shardNodesCache.get(shard); + List availableNodes = new LinkedList<>(); + for (SolrHost node : nodes) { + if (this.nodeDisableCache.contains(node) || this.nodeUnavailableCache.contains(node)) + continue; + + ShardInstance shardNode = ShardInstance.from(shard, node); + if (this.shardInstanceDisableCache.contains(shardNode) || this.shardInstanceUnavailableCache.contains(shardNode)) + continue; + + availableNodes.add(node); + } + + SolrHost randomNode = availableNodes.get(this.random.nextInt(availableNodes.size())); + + shardNodes.add(ShardInstance.from(shard, randomNode).toAlfrescoModel(shard.toAlfrescoModel(shardSet.toAlfrescoModel()))); + } + + return shardNodes; + } + +} diff --git a/community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java b/community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java new file mode 100644 index 0000000..78905c5 --- /dev/null +++ b/community-module/src/main/java/com/inteligr8/alfresco/asie/util/ShardSetSearchComparator.java @@ -0,0 +1,79 @@ +package com.inteligr8.alfresco.asie.util; + +import java.util.Comparator; + +import org.alfresco.repo.index.shard.ShardMethodEnum; + +import com.inteligr8.alfresco.asie.model.ShardSet; + +public class ShardSetSearchComparator implements Comparator { + + @Override + public int compare(ShardSet ss1, ShardSet ss2) { + int compare = this.compare(ss1.getMethod(), ss2.getMethod()); + if (compare != 0) + return compare; + + return this.compare(ss1.getShards(), ss2.getShards()); + } + + private int compare(ShardMethodEnum method1, ShardMethodEnum method2) { + if (method1.equals(method2)) + return 0; + + switch (method1) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + return -1; + case PROPERTY: + case DATE: + switch (method2) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + return 1; + default: + return -1; + } + case ACL_ID: + case MOD_ACL_ID: + switch (method2) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + case PROPERTY: + case DATE: + return 1; + default: + return -1; + } + default: + switch (method2) { + case EXPLICIT_ID: + case EXPLICIT_ID_FALLBACK_LRIS: + case PROPERTY: + case DATE: + case ACL_ID: + case MOD_ACL_ID: + return 1; + default: + } + } + + return 0; + } + + private int compare(Short shards1, Short shards2) { + // the larger the shard count, the more shards that may need to be queried + // so prefer smaller shard counts + // no shard count (DB_ID_RANGE) should be treated as the worst (unlimited) + if (shards1 == null && shards2 == null) { + return 0; + } else if (shards1 == null) { + return 1; + } else if (shards2 == null) { + return -1; + } else { + return shards1.compareTo(shards2); + } + } + +} diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties new file mode 100644 index 0000000..db2025b --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/alfresco-global.properties @@ -0,0 +1,109 @@ + +inteligr8.asie.registerUnknownShardDisabled=false +inteligr8.asie.offlineIdleShardInSeconds=120 +inteligr8.asie.forgetOfflineShardInSeconds=86400 + + + +# we don't want items expiring out of the following caches +# an evicition policy of NONE disables the maxItems limits + +# Overrides of alfresco-repository.jar/alfresco/caches.properties +cache.asieShardsetsSharedCache.tx.maxItems=65536 +cache.asieShardsetsSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardsetsSharedCache.maxItems=65536 +cache.asieShardsetsSharedCache.timeToLiveSeconds=0 +cache.asieShardsetsSharedCache.maxIdleSeconds=0 +cache.asieShardsetsSharedCache.cluster.type=fully-distributed +cache.asieShardsetsSharedCache.backup-count=1 +cache.asieShardsetsSharedCache.eviction-policy=NONE +cache.asieShardsetsSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardsetsSharedCache.readBackupData=false + +cache.asieNodesSharedCache.tx.maxItems=65536 +cache.asieNodesSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodesSharedCache.maxItems=65536 +cache.asieNodesSharedCache.timeToLiveSeconds=0 +cache.asieNodesSharedCache.maxIdleSeconds=0 +cache.asieNodesSharedCache.cluster.type=fully-distributed +cache.asieNodesSharedCache.backup-count=1 +cache.asieNodesSharedCache.eviction-policy=NONE +cache.asieNodesSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodesSharedCache.readBackupData=false + +cache.asieShardNodesSharedCache.tx.maxItems=65536 +cache.asieShardNodesSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardNodesSharedCache.maxItems=65536 +cache.asieShardNodesSharedCache.timeToLiveSeconds=0 +cache.asieShardNodesSharedCache.maxIdleSeconds=0 +cache.asieShardNodesSharedCache.cluster.type=fully-distributed +cache.asieShardNodesSharedCache.backup-count=1 +cache.asieShardNodesSharedCache.eviction-policy=NONE +cache.asieShardNodesSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardNodesSharedCache.readBackupData=false + +cache.asieShardInstanceStateSharedCache.tx.maxItems=65536 +cache.asieShardInstanceStateSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceStateSharedCache.maxItems=65536 +cache.asieShardInstanceStateSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceStateSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceStateSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceStateSharedCache.backup-count=1 +cache.asieShardInstanceStateSharedCache.eviction-policy=NONE +cache.asieShardInstanceStateSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceStateSharedCache.readBackupData=false + +cache.asieNodeDisabledSharedCache.tx.maxItems=65536 +cache.asieNodeDisabledSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodeDisabledSharedCache.maxItems=65536 +cache.asieNodeDisabledSharedCache.timeToLiveSeconds=0 +cache.asieNodeDisabledSharedCache.maxIdleSeconds=0 +cache.asieNodeDisabledSharedCache.cluster.type=fully-distributed +cache.asieNodeDisabledSharedCache.backup-count=1 +cache.asieNodeDisabledSharedCache.eviction-policy=NONE +cache.asieNodeDisabledSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodeDisabledSharedCache.readBackupData=false + +cache.asieNodeUnavailableSharedCache.tx.maxItems=65536 +cache.asieNodeUnavailableSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieNodeUnavailableSharedCache.maxItems=65536 +cache.asieNodeUnavailableSharedCache.timeToLiveSeconds=0 +cache.asieNodeUnavailableSharedCache.maxIdleSeconds=0 +cache.asieNodeUnavailableSharedCache.cluster.type=fully-distributed +cache.asieNodeUnavailableSharedCache.backup-count=1 +cache.asieNodeUnavailableSharedCache.eviction-policy=NONE +cache.asieNodeUnavailableSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieNodeUnavailableSharedCache.readBackupData=false + +cache.asieShardInstanceDisabledSharedCache.tx.maxItems=65536 +cache.asieShardInstanceDisabledSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceDisabledSharedCache.maxItems=65536 +cache.asieShardInstanceDisabledSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceDisabledSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceDisabledSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceDisabledSharedCache.backup-count=1 +cache.asieShardInstanceDisabledSharedCache.eviction-policy=NONE +cache.asieShardInstanceDisabledSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceDisabledSharedCache.readBackupData=false + +cache.asieShardInstanceUnavailableSharedCache.tx.maxItems=65536 +cache.asieShardInstanceUnavailableSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieShardInstanceUnavailableSharedCache.maxItems=65536 +cache.asieShardInstanceUnavailableSharedCache.timeToLiveSeconds=0 +cache.asieShardInstanceUnavailableSharedCache.maxIdleSeconds=0 +cache.asieShardInstanceUnavailableSharedCache.cluster.type=fully-distributed +cache.asieShardInstanceUnavailableSharedCache.backup-count=1 +cache.asieShardInstanceUnavailableSharedCache.eviction-policy=NONE +cache.asieShardInstanceUnavailableSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieShardInstanceUnavailableSharedCache.readBackupData=false + +cache.asieCoreExplicitSharedCache.tx.maxItems=65536 +cache.asieCoreExplicitSharedCache.tx.statsEnabled=${caches.tx.statsEnabled} +cache.asieCoreExplicitSharedCache.maxItems=65536 +cache.asieCoreExplicitSharedCache.timeToLiveSeconds=0 +cache.asieCoreExplicitSharedCache.maxIdleSeconds=0 +cache.asieCoreExplicitSharedCache.cluster.type=fully-distributed +cache.asieCoreExplicitSharedCache.backup-count=1 +cache.asieCoreExplicitSharedCache.eviction-policy=NONE +cache.asieCoreExplicitSharedCache.merge-policy=com.hazelcast.map.merge.PutIfAbsentMapMergePolicy +cache.asieCoreExplicitSharedCache.readBackupData=false diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/log4j2.properties new file mode 100644 index 0000000..e69de29 diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml new file mode 100644 index 0000000..444172a --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module-context.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties new file mode 100644 index 0000000..6c5ec20 --- /dev/null +++ b/community-module/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-community-platform-module/module.properties @@ -0,0 +1,11 @@ +module.id=com_inteligr8_alfresco_${project.artifactId} +module.aliases= +module.title=${project.name} +module.description=${project.description} +module.version=${module.version} + +module.repo.version.min=23.0 + +# this is creating all sorts of problems; probably because of the non-standard versioning +module.depends.com.inteligr8.alfresco.cachext-platform-module=* +module.depends.com.inteligr8.alfresco.cxf-jaxrs-platform-module=* diff --git a/community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java b/community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java new file mode 100644 index 0000000..1e05d44 --- /dev/null +++ b/community-module/src/test/java/com/inteligr8/alfresco/asie/QueryConstraintUnitTest.java @@ -0,0 +1,146 @@ +package com.inteligr8.alfresco.asie; + +import java.util.ArrayList; +import java.util.List; + +import org.alfresco.repo.search.impl.parsers.FTSLexer; +import org.alfresco.repo.search.impl.parsers.FTSParser; +import org.alfresco.service.cmr.search.SearchParameters.Operator; +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; + +public class QueryConstraintUnitTest { + + private static final ObjectMapper om = new ObjectMapper(); + + @BeforeClass + public static void init() { + SimpleModule module = new SimpleModule(); + module.addSerializer(Tree.class, new TreeSerializer()); + om.registerModule(module); + } + + @Test + public void testSingleExactTerm() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("=@cm:title:test", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "EXACT_TERM", "test"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleFuzzyTerm() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("@cm:title:test", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "TERM", "test"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleFuzzyString() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("@cm:title:'testing'", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "PHRASE", "'testing'"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleFuzzyStringDoubleQuotes() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("cm:title:\"testing\"", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "PHRASE", "\"testing\""); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testSingleRange() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("@cm:created:[NOW TO '2025-01-01T00:00:00'>", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + tree = this.validateChildren(tree, "CONJUNCTION"); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "RANGE", "INCLUSIVE", "NOW", "'2025-01-01T00:00:00'", "EXCLUSIVE"); + tree = this.validateChildren(tree, "FIELD_REF", "created"); + this.validate(tree, "PREFIX", "cm"); + } + + @Test + public void testTwoTerms() throws RecognitionException, JsonProcessingException { + Tree tree = this.parseFts("=@cm:title:test1 AND @cm:author:test2", Operator.AND); + tree = this.validateChildren(tree, "DISJUNCTION"); + List trees = this.validateChildren(tree, "CONJUNCTION", 2); + + tree = trees.get(0); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "EXACT_TERM", "test1"); + tree = this.validateChildren(tree, "FIELD_REF", "title"); + this.validate(tree, "PREFIX", "cm"); + + tree = trees.get(1); + tree = this.validateChildren(tree, "DEFAULT"); + tree = this.validateChildren(tree, "TERM", "test2"); + tree = this.validateChildren(tree, "FIELD_REF", "author"); + this.validate(tree, "PREFIX", "cm"); + } + + protected void validate(Tree tree, String text, String... extraValues) { + Assert.assertNotNull(tree); + Assert.assertEquals(text, tree.getText()); + Assert.assertEquals(extraValues.length, tree.getChildCount()); + for (int c = 0; c < extraValues.length; c++) + Assert.assertEquals(extraValues[c], tree.getChild(c).getText()); + } + + protected Tree validateChildren(Tree tree, String text, String... extraValues) { + Assert.assertNotNull(tree); + Assert.assertEquals(text, tree.getText()); + Assert.assertEquals(extraValues.length + 1, tree.getChildCount()); + for (int c = 0; c < extraValues.length; c++) + Assert.assertEquals(extraValues[c], tree.getChild(c).getText()); + return tree.getChild(extraValues.length); + } + + protected List validateChildren(Tree tree, String text, int count) { + Assert.assertNotNull(tree); + Assert.assertEquals(text, tree.getText()); + Assert.assertEquals(count, tree.getChildCount()); + List children = new ArrayList<>(); + for (int c = 0; c < tree.getChildCount(); c++) + children.add(tree.getChild(c)); + return children; + } + + protected Tree parseFts(String ftsQuery, Operator defaultOperator) throws RecognitionException, JsonProcessingException { + CharStream cs = new ANTLRStringStream(ftsQuery); + FTSLexer lexer = new FTSLexer(cs); + CommonTokenStream tokens = new CommonTokenStream(lexer); + FTSParser parser = new FTSParser(tokens); + parser.setDefaultFieldConjunction(defaultOperator.equals(Operator.AND)); + parser.setMode(defaultOperator.equals(Operator.AND) ? FTSParser.Mode.DEFAULT_CONJUNCTION : FTSParser.Mode.DEFAULT_DISJUNCTION); + CommonTree tree = (CommonTree) parser.ftsQuery().getTree(); + return tree; + } + +} diff --git a/community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java b/community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java new file mode 100644 index 0000000..b9da03a --- /dev/null +++ b/community-module/src/test/java/com/inteligr8/alfresco/asie/TreeSerializer.java @@ -0,0 +1,44 @@ +package com.inteligr8.alfresco.asie; + +import java.io.IOException; + +import org.antlr.runtime.tree.Tree; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +public class TreeSerializer extends StdSerializer { + + private static final long serialVersionUID = -2714782538361726878L; + + public TreeSerializer() { + super(Tree.class); + } + + public TreeSerializer(Class type) { + super(type); + } + + public TreeSerializer(JavaType type) { + super(type); + } + + @Override + public void serialize(Tree value, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeStartObject(); + if (value.getText() != null) + gen.writeStringField("text", value.getText()); + + if (value.getChildCount() > 0) { + gen.writeArrayFieldStart("children"); + for (int c = 0; c < value.getChildCount(); c++) + gen.writeObject(value.getChild(c)); + gen.writeEndArray(); + } + + gen.writeEndObject(); + } + +} diff --git a/enterprise-module/metadata.keystore b/enterprise-module/metadata.keystore new file mode 100644 index 0000000..2c2a1d9 Binary files /dev/null and b/enterprise-module/metadata.keystore differ diff --git a/enterprise-module/pom.xml b/enterprise-module/pom.xml index e51e470..663ba9e 100644 --- a/enterprise-module/pom.xml +++ b/enterprise-module/pom.xml @@ -16,8 +16,12 @@ ASIE Platform Module for ACS Enterprise - 5.2.0 + 4.9.0 23.3.0 + 23.3.0.98 + 10-2.1 + + true @@ -80,7 +84,7 @@ com.inteligr8.alfresco cxf-jaxrs-platform-module 1.3.1-acs-v23.3 - provided + amp diff --git a/enterprise-module/rad.sh b/enterprise-module/rad.sh index 7cb0a80..3c7c2fa 100644 --- a/enterprise-module/rad.sh +++ b/enterprise-module/rad.sh @@ -1,22 +1,22 @@ #!/bin/sh discoverArtifactId() { - ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate` + ARTIFACT_ID=`mvn -q -Dexpression=project.artifactId -DforceStdout help:evaluate | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g'` } rebuild() { echo "Rebuilding project ..." - mvn process-classes + mvn process-test-classes } start() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad process-classes + mvn -Drad process-test-classes } start_log() { echo "Rebuilding project and starting Docker containers to support rapid application development ..." - mvn -Drad -Ddocker.showLogs process-classes + mvn -Drad -Ddocker.showLogs process-test-classes } stop() { diff --git a/pom.xml b/pom.xml index 9bb5855..cc1df4d 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ asie-api shared enterprise-module + community-module diff --git a/shared/pom.xml b/shared/pom.xml index 468ae65..0dee741 100644 --- a/shared/pom.xml +++ b/shared/pom.xml @@ -16,7 +16,7 @@ ASIE Shared Library for Platform Modules - 5.2.0 + 4.9.0 23.3.0 diff --git a/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/backupNode.get.desc.xml b/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/backupNode.get.desc.xml index b7145c6..87e2c02 100644 --- a/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/backupNode.get.desc.xml +++ b/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/backupNode.get.desc.xml @@ -8,10 +8,8 @@

Retrieve a reference to the ASIE node that should be used for the backup of the specified ASIE shard registered with ACS.

The following path parameters are expected:

-
shardSet
-
A shard method combined with its distinguishing properties; - methods: MOD_ACL_ID, ACL_ID, DB_ID, DB_ID_RANGE, DATE, PROPERTY, EXPLICIT_ID; - e.g. PROPERTY;key:cm:created;regex:^d{4} or DB_ID
+
shardCore
+
A core name (prefix) for the ASIE shard (e.g. alfresco)
shardId
A number starting at 1
@@ -34,7 +32,7 @@ ]]> - /inteligr8/asie/shard/{shardSet}/{shardId}/backup + /inteligr8/asie/shard/{shardCore}/{shardId}/backup any diff --git a/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/leadNode.get.desc.xml b/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/leadNode.get.desc.xml index 1008ff7..c1e839f 100644 --- a/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/leadNode.get.desc.xml +++ b/shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/leadNode.get.desc.xml @@ -8,10 +8,8 @@

Retrieve a reference to the most current/up-to-date ASIE node for the specified ASIE shard registered with ACS.

The following path parameters are expected:

-
shardSet
-
A shard method combined with its distinguishing properties; - methods: MOD_ACL_ID, ACL_ID, DB_ID, DB_ID_RANGE, DATE, PROPERTY, EXPLICIT_ID; - e.g. PROPERTY;key:cm:created;regex:^d{4} or DB_ID
+
shardCore
+
A core name (prefix) for the ASIE shard (e.g. alfresco)
shardId
A number starting at 1
@@ -30,7 +28,7 @@ ]]> - /inteligr8/asie/shard/{shardSet}/{shardId}/lead + /inteligr8/asie/shard/{shardCore}/{shardId}/lead any