Compare commits

..
20 Commits
Author SHA1 Message Date
brian.long ff068d8d3f v1.3.3 poms 2026-01-12 15:05:11 -05:00
brian.long 354ea3f337 Merge branch 'develop' into stable 2026-01-12 15:04:52 -05:00
brian.long ed2251065d added sample versions for enterprise deps 2026-01-12 15:04:40 -05:00
brian.long 1aa133876a optimizing property-based shard method 2026-01-12 15:02:10 -05:00
brian.long b737bccd1c Merge branch 'develop' into stable 2026-01-12 12:19:55 -05:00
brian.long 18ca6f09c5 move provided libs to deps 2026-01-12 12:19:44 -05:00
brian.long ed97efb4ed Merge branch 'develop' into stable 2026-01-12 12:16:03 -05:00
brian.long 59e745706b fix POM version specs 2026-01-12 12:15:52 -05:00
brian.long b6bae65e03 Merge branch 'develop' into stable 2026-01-12 12:07:53 -05:00
brian.long 52caf9b555 fix javadoc for index changes 2026-01-12 12:07:40 -05:00
brian.long fe95ff6239 set asie-api version v1.1.1-asie2 2026-01-12 12:07:26 -05:00
brian.long a1144fcad4 Merge branch 'develop' into stable 2026-01-12 11:58:42 -05:00
brian.long 560e147333 nexus-staging to central-publishing plugin; upgraded plugin versions 2026-01-12 11:58:36 -05:00
brian.long 299be529ba v1.3.2 poms 2026-01-12 11:55:16 -05:00
brian.long ff331cac48 v1.1.1-asie2 pom 2026-01-12 11:53:17 -05:00
brian.long d86c96fcb1 Merge branch 'develop' into stable 2026-01-12 11:52:20 -05:00
brian.long 29c4b4e1c4 increased debug/trace logging during shard discovery 2026-01-12 11:51:43 -05:00
brian.long 2c3fc8495d add index support to reconcile 2026-01-12 11:51:28 -05:00
brian.long bffceee21c added index endpoint 2026-01-12 11:50:19 -05:00
brian.long 294b684292 added ASIE API for index action 2026-01-12 11:29:44 -05:00
16 changed files with 477 additions and 113 deletions
+2 -2
View File
@@ -6,13 +6,13 @@
<parent>
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-platform-module-parent</artifactId>
<version>1.3.1</version>
<version>1.3.3</version>
<relativePath>../</relativePath>
</parent>
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-api</artifactId>
<version>1.1.0-asie2</version>
<version>1.1.1-asie2</version>
<packaging>jar</packaging>
<name>ASIE Jakarta RS API</name>
@@ -7,6 +7,7 @@ import com.inteligr8.alfresco.asie.model.core.DisableIndexingRequest;
import com.inteligr8.alfresco.asie.model.core.EnableIndexingRequest;
import com.inteligr8.alfresco.asie.model.core.FixAction;
import com.inteligr8.alfresco.asie.model.core.FixRequest;
import com.inteligr8.alfresco.asie.model.core.IndexRequest;
import com.inteligr8.alfresco.asie.model.core.IndexingStatusAction;
import com.inteligr8.alfresco.asie.model.core.NewCoreRequest;
import com.inteligr8.alfresco.asie.model.core.NewDefaultIndexRequest;
@@ -61,6 +62,10 @@ public interface CoreAdminApi extends com.inteligr8.solr.api.CoreAdminApi {
@Produces(MediaType.APPLICATION_JSON)
ActionCoreResponse<Action> purge(@BeanParam PurgeRequest request);
@GET
@Produces(MediaType.APPLICATION_JSON)
ActionCoreResponse<Action> index(@BeanParam IndexRequest request);
@GET
@Produces(MediaType.APPLICATION_JSON)
ActionCoreResponse<Action> reindex(@BeanParam ReindexRequest request);
@@ -0,0 +1,106 @@
package com.inteligr8.alfresco.asie.model.core;
import com.inteligr8.solr.model.JsonFormattedResponseRequest;
import jakarta.annotation.Nonnull;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.QueryParam;
public class IndexRequest extends JsonFormattedResponseRequest<IndexRequest> {
private static final String ACTION = "index";
@QueryParam("action")
@DefaultValue(ACTION)
@Nonnull
private String action = ACTION;
@QueryParam("core")
private String core;
@QueryParam("txid")
private Long transactionId;
@QueryParam("acltxid")
private Long aclTransactionId;
@QueryParam("nodeId")
private Long nodeId;
@QueryParam("aclid")
private Long aclId;
public String getAction() {
return action;
}
public void setAction(String action) {
this.action = action;
}
public String getCore() {
return core;
}
public void setCore(String core) {
this.core = core;
}
public IndexRequest withCore(String core) {
this.core = core;
return this;
}
public Long getTransactionId() {
return transactionId;
}
public void setTransactionId(Long transactionId) {
this.transactionId = transactionId;
}
public IndexRequest withTransactionId(Long transactionId) {
this.transactionId = transactionId;
return this;
}
public Long getAclTransactionId() {
return aclTransactionId;
}
public void setAclTransactionId(Long aclTransactionId) {
this.aclTransactionId = aclTransactionId;
}
public IndexRequest withAclTransactionId(Long aclTransactionId) {
this.aclTransactionId = aclTransactionId;
return this;
}
public Long getNodeId() {
return nodeId;
}
public void setNodeId(Long nodeId) {
this.nodeId = nodeId;
}
public IndexRequest withNodeId(Long nodeId) {
this.nodeId = nodeId;
return this;
}
public Long getAclId() {
return aclId;
}
public void setAclId(Long aclId) {
this.aclId = aclId;
}
public IndexRequest withAclId(Long aclId) {
this.aclId = aclId;
return this;
}
}
+33 -30
View File
@@ -6,7 +6,7 @@
<parent>
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-platform-module-parent</artifactId>
<version>1.3.1</version>
<version>1.3.3</version>
<relativePath>../</relativePath>
</parent>
@@ -33,35 +33,6 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Provided by cxf-jaxrs-platform-module, but packaged due to shared -->
<dependency>
<groupId>com.inteligr8</groupId>
<artifactId>common-rest-client</artifactId>
<scope>provided</scope>
</dependency>
<!-- Provided by cxf-jaxrs-platform-module, but packaged due to solr-api -->
<dependency>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jakarta-xmlbind-annotations</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
</dependencyManagement>
@@ -86,6 +57,38 @@
<version>1.3.2-acs-v23.3</version>
<type>amp</type>
</dependency>
<!-- Provided by cxf-jaxrs-platform-module, but packaged due to shared -->
<dependency>
<groupId>com.inteligr8</groupId>
<artifactId>common-rest-client</artifactId>
<scope>provided</scope>
</dependency>
<!-- Provided by cxf-jaxrs-platform-module, but packaged due to solr-api -->
<dependency>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
<version>4.0.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>2.17.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jakarta-xmlbind-annotations</artifactId>
<version>2.17.2</version>
<scope>provided</scope>
</dependency>
<!-- Including for testing purposes only -->
<dependency>
@@ -50,12 +50,15 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
for (Entry<Shard, Set<ShardState>> flocShard : floc.getValue().entrySet()) {
for (ShardState shardState : flocShard.getValue()) {
ShardSet shardSet = ShardSet.from(floc.getKey(), shardState);
if (shardSet.getCore().equals(core))
if (shardSet.getCore().equals(core)) {
this.logger.debug("Found matching shard set: {}: {}", core, shardSet);
return shardSet;
}
}
}
}
this.logger.trace("Did not find matching shard set: {}", core);
return null;
}
@@ -96,7 +99,8 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
}
}
}
this.logger.trace("Did not find matching node: {}:{}", nodeHostname, nodePort);
return null;
}
@@ -127,6 +131,7 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
ShardInstance shardInstance = shardState.getShardInstance();
if (node.equals(SolrHost.from(shardInstance))) {
this.logger.trace("Found shard on node: {}: {}", node, shardInstance.getShard().getInstance());
Map<Integer, ShardInstanceState> shardStates = setShardStates.get(shardSet);
if (shardStates == null)
setShardStates.put(shardSet, shardStates = new HashMap<>());
@@ -152,7 +157,9 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
for (Entry<Floc, Map<Shard, Set<ShardState>>> floc : flocs.entrySet()) {
if (shardMethodSet.contains(floc.getKey().getShardMethod())) {
this.logger.trace("Found qualifying shard method: {}", floc.getKey().getShardMethod());
ShardState shardState = this.extractAnyShardState(floc.getValue());
this.logger.trace("Found single shard state representing shard method: {}: {}", floc.getKey().getShardMethod(), shardState);
shardSets.add(ShardSet.from(floc.getKey(), shardState));
}
}
+27 -16
View File
@@ -5,7 +5,7 @@
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-platform-module-parent</artifactId>
<version>1.3.1</version>
<version>1.3.3</version>
<packaging>pom</packaging>
<name>ASIE Platform Module Parent</name>
@@ -45,13 +45,24 @@
<maven.deploy.skip>true</maven.deploy.skip>
</properties>
<dependencyManagement>
<dependencies>
<!-- Provided by cxf-jaxrs-platform-module, but packaged due to shared -->
<dependency>
<groupId>com.inteligr8</groupId>
<artifactId>common-rest-client</artifactId>
<version>3.0.3-cxf</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<pluginManagement>
<plugins>
<!-- avoids log4j dependency -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.13.0</version>
<version>3.14.1</version>
</plugin>
<!-- avoids struts dependency -->
<plugin>
@@ -61,27 +72,27 @@
<!-- Force use of a new maven-dependency-plugin that doesn't download struts dependency -->
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.8.1</version>
<version>3.9.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.4.0</version>
<version>3.5.4</version>
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.11.0</version>
<version>5.14.0</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.4.0</version>
<version>3.5.4</version>
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.11.0</version>
<version>5.14.0</version>
</dependency>
</dependencies>
</plugin>
@@ -98,7 +109,7 @@
<profiles>
<profile>
<id>ossrh-release</id>
<id>central-publish</id>
<properties>
<maven.deploy.skip>true</maven.deploy.skip>
</properties>
@@ -138,20 +149,20 @@
</executions>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.7.0</version>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<version>0.8.0</version>
<extensions>true</extensions>
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
<publishingServerId>central</publishingServerId>
<autoPublish>true</autoPublish>
</configuration>
<!-- for some reason this is required... -->
<executions>
<execution>
<id>ossrh-deploy</id>
<id>deploy</id>
<phase>deploy</phase>
<goals><goal>deploy</goal></goals>
<goals><goal>publish</goal></goals>
</execution>
</executions>
</plugin>
+2 -3
View File
@@ -6,7 +6,7 @@
<parent>
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-platform-module-parent</artifactId>
<version>1.3.1</version>
<version>1.3.3</version>
<relativePath>../</relativePath>
</parent>
@@ -36,12 +36,11 @@
<dependency>
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-api</artifactId>
<version>1.1.0-asie2</version>
<version>1.1.1-asie2</version>
</dependency>
<dependency>
<groupId>com.inteligr8</groupId>
<artifactId>common-rest-client</artifactId>
<version>3.0.3-cxf</version>
</dependency>
<!-- Needed by this module, but provided by ACS -->
@@ -0,0 +1,24 @@
package com.inteligr8.alfresco.asie.rest;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.service.IndexService;
import com.inteligr8.alfresco.asie.spi.ActionCallback;
@Component(value = "webscript.com.inteligr8.alfresco.asie.indexAcsNode.post")
public class IndexAcsNodeWebScript extends AbstractAcsNodeActionWebScript {
@Autowired
private IndexService indexSerivce;
@Override
protected void executeAction(long nodeDbId, ActionCallback callback, long fullQueueTimeout, TimeUnit fullQueueUnit,
long execTimeout, TimeUnit execUnit) throws TimeoutException, InterruptedException {
this.indexSerivce.index(nodeDbId, callback, 10L, TimeUnit.SECONDS, 30L, TimeUnit.SECONDS);
}
}
@@ -32,6 +32,7 @@ public class ReconcileAcsNodesWebScript extends AbstractAsieWebScript {
public void executeAuthorized(WebScriptRequest request, WebScriptResponse response) throws IOException {
final int fromDbId = this.getRequestTemplateIntegerVariable(request, "fromDbId");
final int toDbId = this.getRequestTemplateIntegerVariable(request, "toDbId");
final boolean index = Boolean.TRUE.equals(this.getOptionalQueryParameter(request, "index", Boolean.class));
final boolean reindex = Boolean.TRUE.equals(this.getOptionalQueryParameter(request, "reindex", Boolean.class));
final boolean includeReconciled = Boolean.TRUE.equals(this.getOptionalQueryParameter(request, "includeReconciled", Boolean.class));
@@ -101,7 +102,7 @@ public class ReconcileAcsNodesWebScript extends AbstractAsieWebScript {
};
try {
this.reconcileService.reconcile(fromDbId, toDbId, null, reindex, callback, 1L, TimeUnit.HOURS, 2L, TimeUnit.MINUTES);
this.reconcileService.reconcile(fromDbId, toDbId, null, index, reindex, callback, 1L, TimeUnit.HOURS, 2L, TimeUnit.MINUTES);
if (responseMap.containsKey("error")) {
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
@@ -11,6 +11,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.index.shard.Floc;
@@ -18,6 +19,8 @@ import org.alfresco.repo.index.shard.Shard;
import org.alfresco.repo.index.shard.ShardInstance;
import org.alfresco.repo.index.shard.ShardRegistry;
import org.alfresco.repo.index.shard.ShardState;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.cmr.search.SearchService;
@@ -48,12 +51,18 @@ public abstract class AbstractNodeActionService {
@Autowired
private NamespaceService namespaceService;
@Autowired
private NodeService nodeService;
@Autowired
private ApiService apiService;
@Autowired
private ExecutorManager executorManager;
@Autowired
private SolrShardHashService shardHashService;
@Autowired(required = false)
@Qualifier(Constants.QUALIFIER_ASIE)
private ShardRegistry shardRegistry;
@@ -216,42 +225,56 @@ public abstract class AbstractNodeActionService {
List<com.inteligr8.alfresco.asie.model.ShardInstance> instances = new LinkedList<>();
List<ShardInstance> slicedInstances = this.shardRegistry.getIndexSlice(searchParams);
if (slicedInstances != null) {
this.logger.trace("Due to a sharding method, considering only applicable shards and their ASIE nodes: {}: {}", nodeDbId, slicedInstances);
for (ShardInstance instance : slicedInstances)
instances.add(this.toModel(instance));
} else {
for (Entry<Floc, Map<Shard, Set<ShardState>>> floc : this.shardRegistry.getFlocs().entrySet()) {
if (!floc.getKey().getStoreRefs().contains(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE))
continue;
for (Entry<Shard, Set<ShardState>> shard : floc.getValue().entrySet()) {
for (ShardState shardState : shard.getValue())
// we need a ShardRegistry method like getIndexSlice, but
// (1) works with shard methods other than explicit
// (2) returns all possible instances/nodes, not just one node per instance
for (Entry<Floc, Map<Shard, Set<ShardState>>> floc : this.shardRegistry.getFlocs().entrySet()) {
if (!floc.getKey().getStoreRefs().contains(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE))
continue;
Integer shardHash = null;
ShardSet shardset = null;
for (Entry<Shard, Set<ShardState>> shard : floc.getValue().entrySet()) {
for (ShardState shardState : shard.getValue()) {
// every shard and instance (state) in the FLOC is the same
// but we need one ShardState to determine the full configuration
// so we are computing the shardHash once and caching it
if (shardset == null) {
shardset = ShardSet.from(floc.getKey(), shardState);
switch (shardset.getMethod()) {
case PROPERTY:
this.logger.trace("Using property-based sharding method; discovering target shard ...");
NodeRef nodeRef = this.nodeService.getNodeRef(nodeDbId);
Object propValue = this.nodeService.getProperty(nodeRef, QName.createQName(shardset.getPrefixedProperty(), this.namespaceService));
if (propValue != null) {
this.logger.trace("Discovered node property for sharding: {} <=> {} => {}", nodeDbId, nodeRef, propValue);
Matcher matcher = shardset.getRegex().matcher(propValue.toString());
String hashable = matcher.group(1);
this.logger.trace("Extracted shardable value from node: {} <=> {} => {}", nodeDbId, propValue, hashable);
shardHash = this.shardHashService.hash(hashable, shardset.getShards().intValue());
this.logger.debug("Hash shardable value to shard instance ID: {} <=> {} => {}", nodeDbId, hashable, shardHash);
}
break;
default:
this.logger.trace("Despite sharding, considering all shards and nodes without optimization: {}: {}", nodeDbId, instances);
}
}
if (shardHash != null) {
if (shard.getKey().getInstance() == shardHash)
instances.add(this.toModel(shardState.getShardInstance(), shardState));
} else {
instances.add(this.toModel(shardState.getShardInstance(), shardState));
}
}
}
this.logger.trace("Despite sharding, considering all shards and nodes: {}: {}", nodeDbId, instances);
}
return instances;
}
private com.inteligr8.alfresco.asie.model.ShardInstance toModel(ShardInstance instance) {
// get any random shardState
Floc floc = instance.getShard().getFloc();
Map<Shard, Set<ShardState>> shardsStates = this.shardRegistry.getFlocs().get(floc);
if (shardsStates == null)
throw new IllegalStateException();
Set<ShardState> shardStates = shardsStates.get(instance.getShard());
if (shardStates == null || shardStates.isEmpty())
throw new IllegalStateException();
ShardState anyShardState = shardStates.iterator().next();
return this.toModel(instance, anyShardState);
}
private com.inteligr8.alfresco.asie.model.ShardInstance toModel(ShardInstance instance, ShardState anyShardState) {
Floc floc = instance.getShard().getFloc();
@@ -32,6 +32,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.model.ShardInstance;
import com.inteligr8.alfresco.asie.spi.IndexCallback;
import com.inteligr8.alfresco.asie.spi.ReconcileCallback;
import com.inteligr8.alfresco.asie.spi.ReindexCallback;
import com.inteligr8.alfresco.asie.util.CompositeFuture;
@@ -55,6 +56,9 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
@Autowired
private SearchService searchService;
@Autowired
private IndexService indexService;
@Autowired
private ReindexService reindexService;
@@ -98,7 +102,8 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
*
* @param fromDbId A node database ID, inclusive.
* @param toDbId A node database ID, exclusive.
* @param reindexUnreconciled For nodes not found in Solr, attempt to re-index against all applicable Solr instances.
* @param indexUnreconciled For nodes not found in Solr, attempt to index against all applicable Solr instances.
* @param reindexReconciled For nodes found in Solr, attempt to re-index against all applicable Solr instances.
* @param callback A callback to process multiple returned values from the re-index.
* @param queueTimeout A timeout for how long the calling thread should wait for space on the queue.
* @param queueUnit The time units for the `queueTimeout`.
@@ -109,20 +114,21 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
*/
public void reconcile(
long fromDbId, long toDbId, Integer nodesChunkSize,
boolean reindexUnreconciled,
boolean indexUnreconciled,
boolean reindexReconciled,
ReconcileCallback callback,
long queueTimeout, TimeUnit queueUnit,
long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
if (nodesChunkSize == null)
nodesChunkSize = this.nodesChunkSize;
if (this.logger.isTraceEnabled())
this.logger.trace("reconcile({}, {}, {}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, reindexUnreconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
this.logger.trace("reconcile({}, {}, {}, {}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, indexUnreconciled, reindexReconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
CompositeFuture<Void> future = new CompositeFuture<>();
for (long startDbId = fromDbId; startDbId < toDbId; startDbId += nodesChunkSize) {
long endDbId = Math.min(toDbId, startDbId + nodesChunkSize);
future.combine(this.reconcileChunk(startDbId, endDbId, reindexUnreconciled, callback, queueTimeout, queueUnit, execTimeout, execUnit));
future.combine(this.reconcileChunk(startDbId, endDbId, indexUnreconciled, reindexReconciled, callback, queueTimeout, queueUnit, execTimeout, execUnit));
future.purge(true);
}
@@ -135,18 +141,19 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
public Future<Void> reconcile(
long fromDbId, long toDbId, Integer nodesChunkSize,
boolean reindexUnreconciled,
boolean indexUnreconciled,
boolean reindexReconciled,
ReconcileCallback callback) throws InterruptedException {
if (nodesChunkSize == null)
nodesChunkSize = this.nodesChunkSize;
this.logger.trace("reconcile({}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, reindexUnreconciled);
this.logger.trace("reconcile({}, {}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, indexUnreconciled, reindexReconciled);
CompositeFuture<Void> future = new CompositeFuture<>();
try {
for (long startDbId = fromDbId; startDbId < toDbId; startDbId += nodesChunkSize) {
long endDbId = Math.min(toDbId, startDbId + nodesChunkSize);
future.combine(this.reconcileChunk(startDbId, endDbId, reindexUnreconciled, callback, -1L, null, -1L, null));
future.combine(this.reconcileChunk(startDbId, endDbId, indexUnreconciled, reindexReconciled, callback, -1L, null, -1L, null));
future.purge(true);
}
} catch (TimeoutException te) {
@@ -158,12 +165,13 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
protected Future<Void> reconcileChunk(
long fromDbId, long toDbId,
boolean reindexUnreconciled,
boolean indexUnreconciled,
boolean reindexReconciled,
ReconcileCallback callback,
long queueTimeout, TimeUnit queueUnit,
long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
if (this.logger.isTraceEnabled())
this.logger.trace("reconcileChunk({}, {}, {}, {}, {})", fromDbId, toDbId, reindexUnreconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
this.logger.trace("reconcileChunk({}, {}, {}, {}, {}, {})", fromDbId, toDbId, indexUnreconciled, reindexReconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
int dbIdCount = (int) (toDbId - fromDbId);
@@ -202,22 +210,32 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
final long nodeDbId = _nodeDbId;
this.logger.trace("Attempting to reconcile ACS node: {}", nodeDbId);
Callable<Void> callable;
final int dbIdIndex = (int) (nodeDbId - fromDbId);
if (nodeRefs[dbIdIndex] != null) {
this.logger.trace("A node in the DB is already indexed in Solr: {}: {}", nodeDbId, nodeRefs[dbIdIndex]);
this.reconcileLogger.info("RECONCILED: {} <=> {}", nodeDbId, nodeRefs[dbIdIndex]);
callback.reconciled(nodeDbId);
continue;
callable = new Callable<Void>() {
@Override
public Void call() throws InterruptedException, TimeoutException {
logger.trace("A node in the DB is already indexed in Solr: {}: {}", nodeDbId, nodeRefs[dbIdIndex]);
reconcileLogger.info("RECONCILED: {} <=> {}", nodeDbId, nodeRefs[dbIdIndex]);
callback.reconciled(nodeDbId);
if (reindexReconciled)
reindex(nodeDbId, nodeRefs[dbIdIndex], callback, execTimeout, execUnit);
return null;
}
};
} else {
callable = new Callable<Void>() {
@Override
public Void call() throws InterruptedException, TimeoutException {
reconcile(nodeDbId, indexUnreconciled, callback, execTimeout, execUnit);
return null;
}
};
}
Callable<Void> callable = new Callable<Void>() {
@Override
public Void call() throws InterruptedException, TimeoutException {
reconcile(nodeDbId, reindexUnreconciled, callback, execTimeout, execUnit);
return null;
}
};
if (queueTimeout < 0L) {
future.combine(this.executor.submit(callable, -1L, null));
} else {
@@ -229,7 +247,7 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
}
public void reconcile(long nodeDbId,
boolean reindexUnreconciled,
boolean index,
ReconcileCallback callback,
long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
NodeRef nodeRef = this.nodeService.getNodeRef(nodeDbId);
@@ -250,16 +268,58 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
return;
}
if (!reindexUnreconciled) {
if (!index) {
this.logger.debug("A node in the DB is not indexed in Solr: {}: {}", nodeDbId, nodeRef);
this.reconcileLogger.info("UNRECONCILED: {} <=> {}", nodeDbId, nodeRef);
callback.unreconciled(nodeDbId);
} else {
logger.debug("A node in the DB is not indexed in Solr; attempt to reindex: {}: {}", nodeDbId, nodeRef);
this.reindex(nodeDbId, nodeRef, callback, execTimeout, execUnit);
logger.debug("A node in the DB is not indexed in Solr; attempt to index: {}: {}", nodeDbId, nodeRef);
this.index(nodeDbId, nodeRef, callback, execTimeout, execUnit);
}
}
public void index(long nodeDbId, NodeRef nodeRef,
ReconcileCallback callback,
long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
Set<ShardInstance> syncHosts = new HashSet<>();
Set<ShardInstance> asyncHosts = new HashSet<>();
Map<ShardInstance, String> errorHosts = new HashMap<>();
IndexCallback indexCallback = new IndexCallback() {
@Override
public void success(ShardInstance instance) {
reconcileLogger.info("INDEXED: {} <=> {} in {}", nodeDbId, nodeRef, instance);
syncHosts.add(instance);
}
@Override
public void scheduled(ShardInstance instance) {
reconcileLogger.info("INDEXING: {} <=> {} in {}", nodeDbId, nodeRef, instance);
asyncHosts.add(instance);
}
@Override
public void error(ShardInstance instance, String message) {
reconcileLogger.info("FAILED INDEX: {} <=> {} in {}", nodeDbId, nodeRef, instance);
errorHosts.put(instance, message);
}
};
try {
if (execTimeout < 0L) {
this.indexService.index(nodeDbId, indexCallback).get();
} else {
this.indexService.index(nodeDbId, indexCallback).get(execTimeout, execUnit);
}
} catch (ExecutionException ee) {
throw new RuntimeException("An unexpected exception occurred: " + ee.getMessage(), ee);
}
if (callback != null)
callback.processed(nodeDbId, syncHosts, asyncHosts, errorHosts);
}
public void reindex(long nodeDbId, NodeRef nodeRef,
ReconcileCallback callback,
long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
@@ -271,19 +331,19 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
@Override
public void success(ShardInstance instance) {
reconcileLogger.info("REINDEXED: {} <=> {}", nodeDbId, nodeRef);
reconcileLogger.info("REINDEXED: {} <=> {} in {}", nodeDbId, nodeRef, instance);
syncHosts.add(instance);
}
@Override
public void scheduled(ShardInstance instance) {
reconcileLogger.info("REINDEXING: {} <=> {}", nodeDbId, nodeRef);
reconcileLogger.info("REINDEXING: {} <=> {} in {}", nodeDbId, nodeRef, instance);
asyncHosts.add(instance);
}
@Override
public void error(ShardInstance instance, String message) {
reconcileLogger.info("UNINDEXED: {} <=> {}", nodeDbId, nodeRef);
reconcileLogger.info("FAILED REINDEX: {} <=> {} in {}", nodeDbId, nodeRef, instance);
errorHosts.put(instance, message);
}
};
@@ -0,0 +1,43 @@
package com.inteligr8.alfresco.asie.service;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.api.CoreAdminApi;
import com.inteligr8.alfresco.asie.model.ActionCoreResponse;
import com.inteligr8.alfresco.asie.model.core.IndexRequest;
import com.inteligr8.alfresco.asie.spi.ActionCallback;
import com.inteligr8.solr.model.Action;
@Component
public class IndexService extends AbstractNodeActionService {
@Override
protected String getActionName() {
return "index";
}
@Override
protected String getThreadNamePrefix() {
return "solr-index";
}
@Override
protected ActionCoreResponse<Action> execute(CoreAdminApi api, String core, long nodeDbId) {
IndexRequest apiRequest = new IndexRequest().withCore(core).withNodeId(nodeDbId);
return api.index(apiRequest);
}
public Future<Void> index(long nodeDbId, ActionCallback callback) throws InterruptedException {
return super.action(nodeDbId, callback);
}
public void index(long nodeDbId, ActionCallback callback, long fullQueueTimeout, TimeUnit fullQueueUnit,
long execTimeout, TimeUnit execUnit) throws TimeoutException, InterruptedException {
super.action(nodeDbId, callback, fullQueueTimeout, fullQueueUnit, execTimeout, execUnit);
}
}
@@ -0,0 +1,12 @@
package com.inteligr8.alfresco.asie.spi;
import com.inteligr8.alfresco.asie.model.ShardInstance;
public interface IndexCallback extends ActionCallback {
@Override
default void unknownResult(ShardInstance instance) {
throw new IllegalStateException();
}
}
@@ -0,0 +1,63 @@
<webscript xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://bitbucket.org/!api/2.0/snippets/inteligr8/AzMgbp/80fdd26a6b3769a63cdc6b54bf1f39e378545cf7/files/snippet.txt">
<!-- Naming & Organization -->
<shortname>Index ACS Node in ASIE Indexes</shortname>
<family>Inteligr8 ASIE</family>
<description><![CDATA[
<p>Index the specified ACS node in the ASIE indexes.
This call will attempt to index the ACS node on all applicable Solr nodes.
The index operation could be synchronous or asynchronous and could fail on any Solr node.
If any Solr node failed synchronously in the execution, then expect a status code of 500.
The response body will still be identical to the 200/202 status codes.
If any Solr node is executing the index asynchronously and there are no synchronous failures, then expect a status code of 202.</p>
<p>The following path parameters are supported:</p>
<dl>
<dt>nodeId</dt>
<dd>An ACS node ID.</dd>
</dl>
<p>The following response body should be expected in most cases (200, 202, and 500 status codes):</p>
<pre>
{
"nodeDbId": number,
"success": [
"solrHostSync:8983/solr",
...
],
"scheduled": [
"solrHostAsync:8983/solr",
...
],
"error": [
"solrHostThatFailed:8983/solr": {
"message": "string"
},
...
]
}
</pre>
<p>The following status codes should be expected:</p>
<dl>
<dt>200</dt>
<dd>OK</dd>
<dt>202</dt>
<dd>Accepted</dd>
<dt>400</dt>
<dd>The path or query parameters are invalid</dd>
</dl>
]]></description>
<!-- Endpoint Configuration -->
<url>/inteligr8/asie/acs/node/{nodeId}/index</url>
<format default="json">any</format>
<!-- Security -->
<authentication>user</authentication>
<!-- Functionality -->
<cache>
<never>false</never>
<public>false</public>
</cache>
</webscript>
@@ -16,6 +16,13 @@
<dt>toDbId</dt>
<dd>A DB ID integer for the ending point of a range, exclusive.</dd>
</dl>
<p>The following query parameters are also supported:</p>
<dl>
<dt>index</dt>
<dd>Schedule an `INDEX` action against all unreconciled nodes.</dd>
<dt>reindex</dt>
<dd>Schedule a `REINDEX` action against all reconciled nodes.</dd>
</dl>
<p>The following response body should be expected in most cases (200, 202, and 500 status codes):</p>
<pre>
{
@@ -58,7 +65,7 @@
]]></description>
<!-- Endpoint Configuration -->
<url>/inteligr8/asie/acs/nodes/{fromDbId}/{toDbId}/reconcile?reindex={reindex?}</url>
<url>/inteligr8/asie/acs/nodes/{fromDbId}/{toDbId}/reconcile?index={index?}&amp;reindex={reindex?}</url>
<format default="json">any</format>
<!-- Security -->
+1 -1
View File
@@ -6,7 +6,7 @@
<parent>
<groupId>com.inteligr8.alfresco</groupId>
<artifactId>asie-platform-module-parent</artifactId>
<version>1.3.1</version>
<version>1.3.3</version>
<relativePath>../</relativePath>
</parent>