v1.3.5 poms

asie-api/pom.xml

@@ -6,13 +6,13 @@
 	<parent>
 		<groupId>com.inteligr8.alfresco</groupId>
 		<artifactId>asie-platform-module-parent</artifactId>
-		<version>1.3.0</version>
+		<version>1.3.5</version>
 		<relativePath>../</relativePath>
 	</parent>
 
 	<groupId>com.inteligr8.alfresco</groupId>
 	<artifactId>asie-api</artifactId>
-	<version>1.1.0-asie2</version>
+	<version>1.1.1-asie2</version>
 	<packaging>jar</packaging>
 	
 	<name>ASIE Jakarta RS API</name>

asie-api/src/main/java/com/inteligr8/alfresco/asie/api/CoreAdminApi.java

@@ -7,6 +7,7 @@ import com.inteligr8.alfresco.asie.model.core.DisableIndexingRequest;
 import com.inteligr8.alfresco.asie.model.core.EnableIndexingRequest;
 import com.inteligr8.alfresco.asie.model.core.FixAction;
 import com.inteligr8.alfresco.asie.model.core.FixRequest;
+import com.inteligr8.alfresco.asie.model.core.IndexRequest;
 import com.inteligr8.alfresco.asie.model.core.IndexingStatusAction;
 import com.inteligr8.alfresco.asie.model.core.NewCoreRequest;
 import com.inteligr8.alfresco.asie.model.core.NewDefaultIndexRequest;
@@ -61,6 +62,10 @@ public interface CoreAdminApi extends com.inteligr8.solr.api.CoreAdminApi {
 	@Produces(MediaType.APPLICATION_JSON)
 	ActionCoreResponse<Action> purge(@BeanParam PurgeRequest request);
 
+	@GET
+	@Produces(MediaType.APPLICATION_JSON)
+	ActionCoreResponse<Action> index(@BeanParam IndexRequest request);
+
 	@GET
 	@Produces(MediaType.APPLICATION_JSON)
 	ActionCoreResponse<Action> reindex(@BeanParam ReindexRequest request);

asie-api/src/main/java/com/inteligr8/alfresco/asie/model/core/IndexRequest.java

@@ -0,0 +1,106 @@
+package com.inteligr8.alfresco.asie.model.core;
+
+import com.inteligr8.solr.model.JsonFormattedResponseRequest;
+
+import jakarta.annotation.Nonnull;
+import jakarta.ws.rs.DefaultValue;
+import jakarta.ws.rs.QueryParam;
+
+public class IndexRequest extends JsonFormattedResponseRequest<IndexRequest> {
+	
+	private static final String ACTION = "index";
+	
+	@QueryParam("action")
+	@DefaultValue(ACTION)
+	@Nonnull
+	private String action = ACTION;
+	
+	@QueryParam("core")
+	private String core;
+	
+	@QueryParam("txid")
+	private Long transactionId;
+	
+	@QueryParam("acltxid")
+	private Long aclTransactionId;
+	
+	@QueryParam("nodeId")
+	private Long nodeId;
+	
+	@QueryParam("aclid")
+	private Long aclId;
+	
+	public String getAction() {
+		return action;
+	}
+	
+	public void setAction(String action) {
+		this.action = action;
+	}
+	
+	public String getCore() {
+		return core;
+	}
+	
+	public void setCore(String core) {
+		this.core = core;
+	}
+	
+	public IndexRequest withCore(String core) {
+		this.core = core;
+		return this;
+	}
+	
+	public Long getTransactionId() {
+		return transactionId;
+	}
+	
+	public void setTransactionId(Long transactionId) {
+		this.transactionId = transactionId;
+	}
+	
+	public IndexRequest withTransactionId(Long transactionId) {
+		this.transactionId = transactionId;
+		return this;
+	}
+	
+	public Long getAclTransactionId() {
+		return aclTransactionId;
+	}
+	
+	public void setAclTransactionId(Long aclTransactionId) {
+		this.aclTransactionId = aclTransactionId;
+	}
+	
+	public IndexRequest withAclTransactionId(Long aclTransactionId) {
+		this.aclTransactionId = aclTransactionId;
+		return this;
+	}
+	
+	public Long getNodeId() {
+		return nodeId;
+	}
+	
+	public void setNodeId(Long nodeId) {
+		this.nodeId = nodeId;
+	}
+	
+	public IndexRequest withNodeId(Long nodeId) {
+		this.nodeId = nodeId;
+		return this;
+	}
+	
+	public Long getAclId() {
+		return aclId;
+	}
+	
+	public void setAclId(Long aclId) {
+		this.aclId = aclId;
+	}
+	
+	public IndexRequest withAclId(Long aclId) {
+		this.aclId = aclId;
+		return this;
+	}
+
+}

enterprise-module/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>com.inteligr8.alfresco</groupId>
 		<artifactId>asie-platform-module-parent</artifactId>
-		<version>1.3.0</version>
+		<version>1.3.5</version>
 		<relativePath>../</relativePath>
 	</parent>
 
@@ -33,35 +33,6 @@
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
-			
-			<!-- Provided by cxf-jaxrs-platform-module, but packaged due to shared -->
-			<dependency>
-				<groupId>com.inteligr8</groupId>
-				<artifactId>common-rest-client</artifactId>
-				<scope>provided</scope>
-			</dependency>
-			
-			<!-- Provided by cxf-jaxrs-platform-module, but packaged due to solr-api -->
-			<dependency>
-				<groupId>jakarta.ws.rs</groupId>
-				<artifactId>jakarta.ws.rs-api</artifactId>
-				<scope>provided</scope>
-			</dependency>
-			<dependency>
-				<groupId>jakarta.annotation</groupId>
-				<artifactId>jakarta.annotation-api</artifactId>
-				<scope>provided</scope>
-			</dependency>
-			<dependency>
-				<groupId>com.fasterxml.jackson.datatype</groupId>
-				<artifactId>jackson-datatype-jsr310</artifactId>
-				<scope>provided</scope>
-			</dependency>
-			<dependency>
-				<groupId>com.fasterxml.jackson.module</groupId>
-				<artifactId>jackson-module-jakarta-xmlbind-annotations</artifactId>
-				<scope>provided</scope>
-			</dependency>
 		</dependencies>
 	</dependencyManagement>
 
@@ -86,6 +57,38 @@
 			<version>1.3.2-acs-v23.3</version>
 			<type>amp</type>
 		</dependency>
+			
+		<!-- Provided by cxf-jaxrs-platform-module, but packaged due to shared -->
+		<dependency>
+			<groupId>com.inteligr8</groupId>
+			<artifactId>common-rest-client</artifactId>
+			<scope>provided</scope>
+		</dependency>
+			
+		<!-- Provided by cxf-jaxrs-platform-module, but packaged due to solr-api -->
+		<dependency>
+			<groupId>jakarta.ws.rs</groupId>
+			<artifactId>jakarta.ws.rs-api</artifactId>
+			<version>4.0.0</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>jakarta.annotation</groupId>
+			<artifactId>jakarta.annotation-api</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.datatype</groupId>
+			<artifactId>jackson-datatype-jsr310</artifactId>
+			<version>2.17.3</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.module</groupId>
+			<artifactId>jackson-module-jakarta-xmlbind-annotations</artifactId>
+			<version>2.17.2</version>
+			<scope>provided</scope>
+		</dependency>
 
 		<!-- Including for testing purposes only -->
 		<dependency>

enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/rest/UnloadNodeShardWebScript.java

@@ -1,21 +1,33 @@
 package com.inteligr8.alfresco.asie.enterprise.rest;
 
 import org.alfresco.repo.index.shard.ShardState;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.extensions.webscripts.WebScriptException;
 import org.springframework.extensions.webscripts.WebScriptRequest;
+import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Component;
 
 import com.inteligr8.alfresco.asie.model.ShardSet;
 import com.inteligr8.alfresco.asie.rest.model.NodeShardParameterSet;
+import com.inteligr8.alfresco.asie.spi.ShardDiscoveryService;
 
 @Component(value = "webscript.com.inteligr8.alfresco.asie.nodeShard.delete")
 public class UnloadNodeShardWebScript extends AbstractUnregisterNodeWebScript<NodeShardParameterSet> {
+    
+    @Autowired
+    private ShardDiscoveryService sds;
 	
 	@Override
 	protected NodeShardParameterSet createParameters(WebScriptRequest req, String nodeHostname, int nodePort) {
-		ShardSet shardSet = this.getRequiredPathParameter(req, "shardSet", ShardSet.class);
+		String coreName = this.getRequiredPathParameter(req, "shardCore");
 		int shardId = this.getRequiredPathParameter(req, "shardId", Integer.class);
 		
-		return new NodeShardParameterSet(nodeHostname, nodePort, shardSet, shardId);
+		try {
+			ShardSet shardSet = this.sds.findSetByCore(coreName);
+			return new NodeShardParameterSet(nodeHostname, nodePort, shardSet, shardId);
+		} catch (IllegalArgumentException iae) {
+    		throw new WebScriptException(HttpStatus.BAD_REQUEST.value(), iae.getMessage());
+		}
 	}
 	
 	@Override

enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/service/ShardDiscoveryService.java

@@ -50,12 +50,15 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
 			for (Entry<Shard, Set<ShardState>> flocShard : floc.getValue().entrySet()) {
 				for (ShardState shardState : flocShard.getValue()) {
 					ShardSet shardSet = ShardSet.from(floc.getKey(), shardState);
-					if (shardSet.getCore().equals(core))
+					if (shardSet.getCore().equals(core)) {
+						this.logger.debug("Found matching shard set: {}: {}", core, shardSet);
 						return shardSet;
+					}
 				}
 			}
 		}
-		
+
+		this.logger.trace("Did not find matching shard set: {}", core);
 		return null;
     }
     
@@ -96,7 +99,8 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
 				}
 			}
 		}
-		
+
+		this.logger.trace("Did not find matching node: {}:{}", nodeHostname, nodePort);
 		return null;
     }
     
@@ -127,6 +131,7 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
 				
 				ShardInstance shardInstance = shardState.getShardInstance();
 				if (node.equals(SolrHost.from(shardInstance))) {
+					this.logger.trace("Found shard on node: {}: {}", node, shardInstance.getShard().getInstance());
 					Map<Integer, ShardInstanceState> shardStates = setShardStates.get(shardSet);
 					if (shardStates == null)
 						setShardStates.put(shardSet, shardStates = new HashMap<>());
@@ -152,7 +157,9 @@ public class ShardDiscoveryService implements com.inteligr8.alfresco.asie.spi.Sh
 		
 		for (Entry<Floc, Map<Shard, Set<ShardState>>> floc : flocs.entrySet()) {
 			if (shardMethodSet.contains(floc.getKey().getShardMethod())) {
+				this.logger.trace("Found qualifying shard method: {}", floc.getKey().getShardMethod());
 				ShardState shardState = this.extractAnyShardState(floc.getValue());
+				this.logger.trace("Found single shard state representing shard method: {}: {}", floc.getKey().getShardMethod(), shardState);
 				shardSets.add(ShardSet.from(floc.getKey(), shardState));
 			}
 		}

pom.xml

@@ -5,7 +5,7 @@
 	
 	<groupId>com.inteligr8.alfresco</groupId>
 	<artifactId>asie-platform-module-parent</artifactId>
-	<version>1.3.0</version>
+	<version>1.3.5</version>
 	<packaging>pom</packaging>
 	
 	<name>ASIE Platform Module Parent</name>
@@ -45,13 +45,24 @@
 		<maven.deploy.skip>true</maven.deploy.skip>
 	</properties>
 
+	<dependencyManagement>
+		<dependencies>
+			<!-- Provided by cxf-jaxrs-platform-module, but packaged due to shared -->
+			<dependency>
+				<groupId>com.inteligr8</groupId>
+				<artifactId>common-rest-client</artifactId>
+				<version>3.0.3-cxf</version>
+			</dependency>
+		</dependencies>
+	</dependencyManagement>
+
 	<build>
 		<pluginManagement>
 			<plugins>
 				<!-- avoids log4j dependency -->
 				<plugin>
 					<artifactId>maven-compiler-plugin</artifactId>
-					<version>3.13.0</version>
+					<version>3.14.1</version>
 				</plugin>
 				<!-- avoids struts dependency -->
 				<plugin>
@@ -61,27 +72,27 @@
 				<!-- Force use of a new maven-dependency-plugin that doesn't download struts dependency -->
 				<plugin>
 					<artifactId>maven-dependency-plugin</artifactId>
-					<version>3.8.1</version>
+					<version>3.9.0</version>
 				</plugin>
 				<plugin>
 					<artifactId>maven-surefire-plugin</artifactId>
-					<version>3.4.0</version>
+					<version>3.5.4</version>
 					<dependencies>
 						<dependency>
 							<groupId>org.junit.jupiter</groupId>
 							<artifactId>junit-jupiter-engine</artifactId>
-							<version>5.11.0</version>
+							<version>5.14.0</version>
 						</dependency>
 					</dependencies>
 				</plugin>
 				<plugin>
 					<artifactId>maven-failsafe-plugin</artifactId>
-					<version>3.4.0</version>
+					<version>3.5.4</version>
 					<dependencies>
 						<dependency>
 							<groupId>org.junit.jupiter</groupId>
 							<artifactId>junit-jupiter-engine</artifactId>
-							<version>5.11.0</version>
+							<version>5.14.0</version>
 						</dependency>
 					</dependencies>
 				</plugin>
@@ -98,7 +109,7 @@
 
 	<profiles>
 		<profile>
-			<id>ossrh-release</id>
+			<id>central-publish</id>
 			<properties>
 				<maven.deploy.skip>true</maven.deploy.skip>
 			</properties>
@@ -138,20 +149,20 @@
 						</executions>
 					</plugin>
 					<plugin>
-						<groupId>org.sonatype.plugins</groupId>
-						<artifactId>nexus-staging-maven-plugin</artifactId>
-						<version>1.7.0</version>
+						<groupId>org.sonatype.central</groupId>
+						<artifactId>central-publishing-maven-plugin</artifactId>
+						<version>0.8.0</version>
 						<extensions>true</extensions>
 						<configuration>
-							<serverId>ossrh</serverId>
-							<nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
-							<autoReleaseAfterClose>true</autoReleaseAfterClose>
+							<publishingServerId>central</publishingServerId>
+							<autoPublish>true</autoPublish>
 						</configuration>
+						<!-- for some reason this is required... -->
 						<executions>
 							<execution>
-								<id>ossrh-deploy</id>
+								<id>deploy</id>
 								<phase>deploy</phase>
-								<goals><goal>deploy</goal></goals>
+								<goals><goal>publish</goal></goals>
 							</execution>
 						</executions>
 					</plugin>

shared/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>com.inteligr8.alfresco</groupId>
 		<artifactId>asie-platform-module-parent</artifactId>
-		<version>1.3.0</version>
+		<version>1.3.5</version>
 		<relativePath>../</relativePath>
 	</parent>
 
@@ -36,12 +36,11 @@
 		<dependency>
 			<groupId>com.inteligr8.alfresco</groupId>
 			<artifactId>asie-api</artifactId>
-			<version>1.1.0-asie2</version>
+			<version>1.1.1-asie2</version>
 		</dependency>
 		<dependency>
 			<groupId>com.inteligr8</groupId>
 			<artifactId>common-rest-client</artifactId>
-			<version>3.0.3-cxf</version>
 		</dependency>
 		
 		<!-- Needed by this module, but provided by ACS -->

shared/src/main/java/com/inteligr8/alfresco/asie/rest/AbstractActionWebScript.java

@@ -9,14 +9,8 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import org.alfresco.model.ContentModel;
-import org.alfresco.service.cmr.repository.InvalidNodeRefException;
-import org.alfresco.service.cmr.repository.NodeRef;
-import org.alfresco.service.cmr.repository.NodeService;
-import org.alfresco.service.cmr.repository.StoreRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.extensions.webscripts.WebScriptException;
 import org.springframework.extensions.webscripts.WebScriptRequest;
 import org.springframework.extensions.webscripts.WebScriptResponse;

shared/src/main/java/com/inteligr8/alfresco/asie/rest/AbstractAsieNodeShardWebScript.java

@@ -7,8 +7,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.extensions.webscripts.WebScriptRequest;
 import org.springframework.extensions.webscripts.WebScriptResponse;
 
-import com.inteligr8.alfresco.asie.model.ShardSet;
-
 public abstract class AbstractAsieNodeShardWebScript extends AbstractAsieShardableWebScript {
     
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -22,14 +20,14 @@ public abstract class AbstractAsieNodeShardWebScript extends AbstractAsieShardab
 		String nodeHostname = colon < 0 ? nodeEndpoint : nodeEndpoint.substring(0, colon);
 		int nodePort = colon < 0 ? this.getApiService().getDefaultSolrPort() : Integer.parseInt(nodeEndpoint.substring(colon+1));
 
-		ShardSet shardSet = this.getRequiredPathParameter(req, "shardSet", ShardSet.class);
+		String shardCore = this.getRequiredPathParameter(req, "shardCore", String.class);
 		int shardId = this.getRequiredPathParameter(req, "shardId", Integer.class);
 		
-		this.execute(req, res, nodeHostname, nodePort, shardSet, shardId);
+		this.execute(req, res, nodeHostname, nodePort, shardCore, shardId);
     }
     
     protected abstract void execute(WebScriptRequest req, WebScriptResponse res,
-    		String nodeHostname, int nodePort, ShardSet shardSet, int shardId)
+    		String nodeHostname, int nodePort, String shardCore, int shardId)
     				throws IOException;
     
 }

shared/src/main/java/com/inteligr8/alfresco/asie/rest/IndexAcsNodeWebScript.java

@@ -0,0 +1,24 @@
+package com.inteligr8.alfresco.asie.rest;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.inteligr8.alfresco.asie.service.IndexService;
+import com.inteligr8.alfresco.asie.spi.ActionCallback;
+
+@Component(value = "webscript.com.inteligr8.alfresco.asie.indexAcsNode.post")
+public class IndexAcsNodeWebScript extends AbstractAcsNodeActionWebScript {
+	
+	@Autowired
+	private IndexService indexSerivce;
+	
+	@Override
+	protected void executeAction(long nodeDbId, ActionCallback callback, long fullQueueTimeout, TimeUnit fullQueueUnit,
+			long execTimeout, TimeUnit execUnit) throws TimeoutException, InterruptedException {
+		this.indexSerivce.index(nodeDbId, callback, 10L, TimeUnit.SECONDS, 30L, TimeUnit.SECONDS);
+    }
+    
+}

shared/src/main/java/com/inteligr8/alfresco/asie/rest/ReconcileAcsNodesWebScript.java

@@ -32,6 +32,7 @@ public class ReconcileAcsNodesWebScript extends AbstractAsieWebScript {
 	public void executeAuthorized(WebScriptRequest request, WebScriptResponse response) throws IOException {
 		final int fromDbId = this.getRequestTemplateIntegerVariable(request, "fromDbId");
 		final int toDbId = this.getRequestTemplateIntegerVariable(request, "toDbId");
+		final boolean index = Boolean.TRUE.equals(this.getOptionalQueryParameter(request, "index", Boolean.class));
 		final boolean reindex = Boolean.TRUE.equals(this.getOptionalQueryParameter(request, "reindex", Boolean.class));
 		final boolean includeReconciled = Boolean.TRUE.equals(this.getOptionalQueryParameter(request, "includeReconciled", Boolean.class));
 		
@@ -101,7 +102,7 @@ public class ReconcileAcsNodesWebScript extends AbstractAsieWebScript {
 		};
 		
 		try {
-			this.reconcileService.reconcile(fromDbId, toDbId, null, reindex, callback, 1L, TimeUnit.HOURS, 2L, TimeUnit.MINUTES);
+			this.reconcileService.reconcile(fromDbId, toDbId, null, index, reindex, callback, 1L, TimeUnit.HOURS, 2L, TimeUnit.MINUTES);
 
 			if (responseMap.containsKey("error")) {
 				response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());

shared/src/main/java/com/inteligr8/alfresco/asie/service/AbstractActionService.java

@@ -12,17 +12,12 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import org.alfresco.model.ContentModel;
 import org.alfresco.repo.index.shard.Floc;
 import org.alfresco.repo.index.shard.Shard;
 import org.alfresco.repo.index.shard.ShardInstance;
 import org.alfresco.repo.index.shard.ShardRegistry;
 import org.alfresco.repo.index.shard.ShardState;
 import org.alfresco.service.cmr.repository.StoreRef;
-import org.alfresco.service.cmr.search.SearchParameters;
-import org.alfresco.service.cmr.search.SearchService;
-import org.alfresco.service.namespace.NamespaceService;
-import org.alfresco.service.namespace.QName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -45,9 +40,6 @@ public abstract class AbstractActionService {
 
 	private final Logger logger = LoggerFactory.getLogger(this.getClass());
 	
-	@Autowired
-	private NamespaceService namespaceService;
-	
 	@Autowired
 	private ApiService apiService;
 	
@@ -58,10 +50,10 @@ public abstract class AbstractActionService {
 	@Qualifier(Constants.QUALIFIER_ASIE)
 	private ShardRegistry shardRegistry;
 	
-	@Value("${inteligr8.asie.default.concurrentQueueSize:64}")
+	@Value("${inteligr8.asie.default.concurrentQueueSize}")
 	private int concurrentQueueSize;
 	
-	@Value("${inteligr8.asie.default.concurrency:16}")
+	@Value("${inteligr8.asie.default.concurrency}")
 	private int concurrency;
 	
 	protected int getConcurrency() {

shared/src/main/java/com/inteligr8/alfresco/asie/service/AbstractNodeActionService.java

@@ -8,6 +8,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -25,6 +26,7 @@ import org.alfresco.service.namespace.NamespaceService;
 import org.alfresco.service.namespace.QName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
@@ -41,7 +43,7 @@ import com.inteligr8.solr.model.Action;
 import com.inteligr8.solr.model.ActionResponse;
 import com.inteligr8.solr.model.BaseResponse;
 
-public abstract class AbstractNodeActionService {
+public abstract class AbstractNodeActionService implements DisposableBean {
 
 	private final Logger logger = LoggerFactory.getLogger(this.getClass());
 	
@@ -54,14 +56,17 @@ public abstract class AbstractNodeActionService {
 	@Autowired
 	private ExecutorManager executorManager;
 	
+	@Autowired
+	private SolrShardHashService shardHashService;
+	
 	@Autowired(required = false)
 	@Qualifier(Constants.QUALIFIER_ASIE)
 	private ShardRegistry shardRegistry;
 	
-	@Value("${inteligr8.asie.default.concurrentQueueSize:64}")
+	@Value("${inteligr8.asie.default.concurrentQueueSize}")
 	private int concurrentQueueSize;
 	
-	@Value("${inteligr8.asie.default.concurrency:16}")
+	@Value("${inteligr8.asie.default.concurrency}")
 	private int concurrency;
 	
 	protected int getConcurrency() {
@@ -76,6 +81,22 @@ public abstract class AbstractNodeActionService {
 	
 	protected abstract String getActionName();
 	
+	@Override
+	public void destroy() {
+		ExecutorService executor = this.executorManager.get(this.getThreadNamePrefix());
+		if (executor != null) {
+			this.logger.info("Shutting down throttled thread pool executor: {}", this.getThreadNamePrefix());
+			executor.shutdown();
+		}
+	}
+
+	private ThrottledThreadPoolExecutor getExecutor() {
+		return this.executorManager.createThrottled(
+				this.getThreadNamePrefix(),
+				this.getConcurrency(), this.getConcurrency(), this.getConcurrentQueueSize(),
+				1L, TimeUnit.MINUTES);
+	}
+	
 	/**
 	 * This method executes an action on the specified node in Solr using its
 	 * ACS unique database identifier.  The callback handles all the return
@@ -135,11 +156,7 @@ public abstract class AbstractNodeActionService {
 		this.logger.debug("Will attempt to {} ACS node against {} shard instances: {}", this.getActionName(), eligibleInstances.size(), nodeDbId);
 		
 		CompositeFuture<Void> future = new CompositeFuture<>();
-		
-		ThrottledThreadPoolExecutor executor = this.executorManager.createThrottled(
-				this.getThreadNamePrefix(),
-				this.getConcurrency(), this.getConcurrency(), this.getConcurrentQueueSize(),
-				1L, TimeUnit.MINUTES);
+		ThrottledThreadPoolExecutor executor = this.getExecutor();
 		
 		for (final com.inteligr8.alfresco.asie.model.ShardInstance instance : eligibleInstances) {
 			this.logger.trace("Will attempt to {} ACS node against shard instance: {}: {}", this.getActionName(), nodeDbId, instance);
@@ -216,42 +233,40 @@ public abstract class AbstractNodeActionService {
 
 		List<com.inteligr8.alfresco.asie.model.ShardInstance> instances = new LinkedList<>();
 		
-		List<ShardInstance> slicedInstances = this.shardRegistry.getIndexSlice(searchParams);
-		if (slicedInstances != null) {
-			this.logger.trace("Due to a sharding method, considering only applicable shards and their ASIE nodes: {}: {}", nodeDbId, slicedInstances);
-			
-			for (ShardInstance instance : slicedInstances)
-				instances.add(this.toModel(instance));
-		} else {
-			for (Entry<Floc, Map<Shard, Set<ShardState>>> floc : this.shardRegistry.getFlocs().entrySet()) {
-				if (!floc.getKey().getStoreRefs().contains(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE))
-					continue;
-				for (Entry<Shard, Set<ShardState>> shard : floc.getValue().entrySet()) {
-					for (ShardState shardState : shard.getValue())
+		// we need a ShardRegistry method like getIndexSlice, but
+		// (1) works with shard methods other than explicit
+		// (2) returns all possible instances/nodes, not just one node per instance
+		
+		for (Entry<Floc, Map<Shard, Set<ShardState>>> floc : this.shardRegistry.getFlocs().entrySet()) {
+			if (!floc.getKey().getStoreRefs().contains(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE))
+				continue;
+
+			int shardHash = -1;
+			ShardSet shardset = null;
+			for (Entry<Shard, Set<ShardState>> shard : floc.getValue().entrySet()) {
+				for (ShardState shardState : shard.getValue()) {
+					// every shard and instance (state) in the FLOC is the same
+					// but we need one ShardState to determine the full configuration
+					// so we are computing the shardHash once and caching it
+					if (shardset == null) {
+						shardset = ShardSet.from(floc.getKey(), shardState);
+						shardHash = this.shardHashService.computeShardInstanceId(shardset, nodeDbId);
+					}
+					
+					if (shardHash >= 0) {
+						if (shard.getKey().getInstance() == shardHash)
+							instances.add(this.toModel(shardState.getShardInstance(), shardState));
+					} else {
 						instances.add(this.toModel(shardState.getShardInstance(), shardState));
+					}
 				}
 			}
-
-			this.logger.trace("Despite sharding, considering all shards and nodes: {}: {}", nodeDbId, instances);
 		}
 
+
 		return instances;
 	}
 	
-	private com.inteligr8.alfresco.asie.model.ShardInstance toModel(ShardInstance instance) {
-		// get any random shardState
-		Floc floc = instance.getShard().getFloc();
-		Map<Shard, Set<ShardState>> shardsStates = this.shardRegistry.getFlocs().get(floc);
-		if (shardsStates == null)
-			throw new IllegalStateException();
-		Set<ShardState> shardStates = shardsStates.get(instance.getShard());
-		if (shardStates == null || shardStates.isEmpty())
-			throw new IllegalStateException();
-		ShardState anyShardState = shardStates.iterator().next();
-		
-		return this.toModel(instance, anyShardState);
-	}
-	
 	private com.inteligr8.alfresco.asie.model.ShardInstance toModel(ShardInstance instance, ShardState anyShardState) {
 		Floc floc = instance.getShard().getFloc();
 		

shared/src/main/java/com/inteligr8/alfresco/asie/service/AcsReconcileService.java

@@ -1,11 +1,10 @@
 package com.inteligr8.alfresco.asie.service;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -26,19 +25,19 @@ import org.apache.commons.collections4.SetUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.DisposableBean;
-import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import com.inteligr8.alfresco.asie.model.ShardInstance;
+import com.inteligr8.alfresco.asie.spi.IndexCallback;
 import com.inteligr8.alfresco.asie.spi.ReconcileCallback;
 import com.inteligr8.alfresco.asie.spi.ReindexCallback;
 import com.inteligr8.alfresco.asie.util.CompositeFuture;
 import com.inteligr8.alfresco.asie.util.ThrottledThreadPoolExecutor;
 
 @Component
-public class AcsReconcileService implements InitializingBean, DisposableBean {
+public class AcsReconcileService implements DisposableBean {
 
 	private final Logger logger = LoggerFactory.getLogger(this.getClass());
 	private final Logger reconcileLogger = LoggerFactory.getLogger("inteligr8.asie.reconcile");
@@ -55,32 +54,41 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 	@Autowired
 	private SearchService searchService;
 	
+	@Autowired
+	private IndexService indexService;
+	
 	@Autowired
 	private ReindexService reindexService;
 	
-	@Value("${inteligr8.asie.reconciliation.nodesChunkSize:250}")
+	@Autowired
+	private ExecutorManager executorManager;
+	
+	@Value("${inteligr8.asie.reconciliation.nodesChunkSize}")
 	private int nodesChunkSize;
 	
-	@Value("${inteligr8.asie.reconciliation.nodeTimeoutSeconds:10}")
+	@Value("${inteligr8.asie.reconciliation.nodeTimeoutSeconds}")
 	private int nodeTimeoutSeconds;
 	
-	@Value("${inteligr8.asie.reconciliation.concurrentQueueSize:64}")
+	@Value("${inteligr8.asie.reconciliation.concurrentQueueSize}")
 	private int concurrentQueueSize;
 	
-	@Value("${inteligr8.asie.reconciliation.concurrency:2}")
+	@Value("${inteligr8.asie.reconciliation.concurrency}")
 	private int concurrency;
 	
-	private ThrottledThreadPoolExecutor executor;
-	
-	@Override
-	public void afterPropertiesSet() {
-		this.executor = new ThrottledThreadPoolExecutor(this.concurrency, this.concurrency, this.concurrentQueueSize, 1L, TimeUnit.MINUTES, "solr-reconcile");
-		this.executor.prestartAllCoreThreads();
-	}
-	
 	@Override
 	public void destroy() {
-		this.executor.shutdown();
+		ExecutorService executor = this.executorManager.get("solr-reconcile");
+		if (executor != null) {
+			this.logger.info("Shutting down throttled thread pool executor: {}", "solr-reconcile");
+			executor.shutdown();
+		}
+	}
+
+	private ThrottledThreadPoolExecutor getExecutor() {
+		return this.executorManager.createThrottled(
+				"solr-reconcile",
+				this.concurrency, this.concurrency, this.concurrentQueueSize,
+				10L, TimeUnit.SECONDS);
 	}
 	
 	/**
@@ -98,7 +106,8 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 	 * 
 	 * @param fromDbId A node database ID, inclusive.
 	 * @param toDbId A node database ID, exclusive.
-	 * @param reindexUnreconciled For nodes not found in Solr, attempt to re-index against all applicable Solr instances.
+	 * @param indexUnreconciled For nodes not found in Solr, attempt to index against all applicable Solr instances.
+	 * @param reindexReconciled For nodes found in Solr, attempt to re-index against all applicable Solr instances.
 	 * @param callback A callback to process multiple returned values from the re-index.
 	 * @param queueTimeout A timeout for how long the calling thread should wait for space on the queue.
 	 * @param queueUnit The time units for the `queueTimeout`.
@@ -109,20 +118,21 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 	 */
 	public void reconcile(
 			long fromDbId, long toDbId, Integer nodesChunkSize,
-			boolean reindexUnreconciled,
+			boolean indexUnreconciled,
+			boolean reindexReconciled,
 			ReconcileCallback callback,
 			long queueTimeout, TimeUnit queueUnit,
 			long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
 		if (nodesChunkSize == null)
 			nodesChunkSize = this.nodesChunkSize;
 		if (this.logger.isTraceEnabled())
-			this.logger.trace("reconcile({}, {}, {}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, reindexUnreconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
+			this.logger.trace("reconcile({}, {}, {}, {}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, indexUnreconciled, reindexReconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
 
 		CompositeFuture<Void> future = new CompositeFuture<>();
 		
 		for (long startDbId = fromDbId; startDbId < toDbId; startDbId += nodesChunkSize) {
 			long endDbId = Math.min(toDbId, startDbId + nodesChunkSize);
-			future.combine(this.reconcileChunk(startDbId, endDbId, reindexUnreconciled, callback, queueTimeout, queueUnit, execTimeout, execUnit));
+			future.combine(this.reconcileChunk(startDbId, endDbId, indexUnreconciled, reindexReconciled, callback, queueTimeout, queueUnit, execTimeout, execUnit));
 			future.purge(true);
 		}
 		
@@ -135,18 +145,19 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 	
 	public Future<Void> reconcile(
 			long fromDbId, long toDbId, Integer nodesChunkSize,
-			boolean reindexUnreconciled,
+			boolean indexUnreconciled,
+			boolean reindexReconciled,
 			ReconcileCallback callback) throws InterruptedException {
 		if (nodesChunkSize == null)
 			nodesChunkSize = this.nodesChunkSize;
-		this.logger.trace("reconcile({}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, reindexUnreconciled);
+		this.logger.trace("reconcile({}, {}, {}, {}, {})", fromDbId, toDbId, nodesChunkSize, indexUnreconciled, reindexReconciled);
 
 		CompositeFuture<Void> future = new CompositeFuture<>();
 		
 		try {
 			for (long startDbId = fromDbId; startDbId < toDbId; startDbId += nodesChunkSize) {
 				long endDbId = Math.min(toDbId, startDbId + nodesChunkSize);
-				future.combine(this.reconcileChunk(startDbId, endDbId, reindexUnreconciled, callback, -1L, null, -1L, null));
+				future.combine(this.reconcileChunk(startDbId, endDbId, indexUnreconciled, reindexReconciled, callback, -1L, null, -1L, null));
 				future.purge(true);
 			}
 		} catch (TimeoutException te) {
@@ -158,12 +169,13 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 	
 	protected Future<Void> reconcileChunk(
 			long fromDbId, long toDbId,
-			boolean reindexUnreconciled,
+			boolean indexUnreconciled,
+			boolean reindexReconciled,
 			ReconcileCallback callback,
 			long queueTimeout, TimeUnit queueUnit,
 			long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
 		if (this.logger.isTraceEnabled())
-			this.logger.trace("reconcileChunk({}, {}, {}, {}, {})", fromDbId, toDbId, reindexUnreconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
+			this.logger.trace("reconcileChunk({}, {}, {}, {}, {}, {})", fromDbId, toDbId, indexUnreconciled, reindexReconciled, queueUnit.toMillis(queueTimeout), execUnit.toMillis(execTimeout));
 		
 		int dbIdCount = (int) (toDbId - fromDbId);
 		
@@ -197,31 +209,46 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 		}
 		
 		CompositeFuture<Void> future = new CompositeFuture<>();
+		ThrottledThreadPoolExecutor executor = this.getExecutor();
 
 		for (long _nodeDbId = fromDbId; _nodeDbId < toDbId; _nodeDbId++) {
 			final long nodeDbId = _nodeDbId;
 			this.logger.trace("Attempting to reconcile ACS node: {}", nodeDbId);
 
+			Callable<Void> callable;
+
 			final int dbIdIndex = (int) (nodeDbId - fromDbId);
 			if (nodeRefs[dbIdIndex] != null) {
-				this.logger.trace("A node in the DB is already indexed in Solr: {}: {}", nodeDbId, nodeRefs[dbIdIndex]);
-				this.reconcileLogger.info("RECONCILED: {} <=> {}", nodeDbId, nodeRefs[dbIdIndex]);
-				callback.reconciled(nodeDbId);
-				continue;
+				callable = new Callable<Void>() {
+					@Override
+					public Void call() throws InterruptedException, TimeoutException {
+						logger.trace("A node in the DB is already indexed in Solr: {}: {}", nodeDbId, nodeRefs[dbIdIndex]);
+						reconcileLogger.info("RECONCILED: {} <=> {}", nodeDbId, nodeRefs[dbIdIndex]);
+						callback.reconciled(nodeDbId);
+
+						if (reindexReconciled)
+							reindex(nodeDbId, nodeRefs[dbIdIndex], callback);
+							// purposefully forgetting about the returned future
+							// its results will be logged
+							// the reconcile thread will continue independently
+							// the callback will lag
+						return null;
+					}
+				};
+			} else {
+				callable = new Callable<Void>() {
+					@Override
+					public Void call() throws InterruptedException, TimeoutException {
+						reconcile(nodeDbId, indexUnreconciled, callback);
+						return null;
+					}
+				};
 			}
 			
-			Callable<Void> callable = new Callable<Void>() {
-				@Override
-				public Void call() throws InterruptedException, TimeoutException {
-					reconcile(nodeDbId, reindexUnreconciled, callback, execTimeout, execUnit);
-					return null;
-				}
-			};
-			
 			if (queueTimeout < 0L) {
-				future.combine(this.executor.submit(callable, -1L, null));
+				future.combine(executor.submit(callable, -1L, null));
 			} else {
-				future.combine(this.executor.submit(callable, queueTimeout, queueUnit));
+				future.combine(executor.submit(callable, queueTimeout, queueUnit));
 			}
 		}
 		
@@ -229,9 +256,8 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 	}
 	
 	public void reconcile(long nodeDbId,
-			boolean reindexUnreconciled,
-			ReconcileCallback callback,
-			long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
+			boolean index,
+			ReconcileCallback callback) throws InterruptedException, TimeoutException {
 		NodeRef nodeRef = this.nodeService.getNodeRef(nodeDbId);
 		if (nodeRef == null) {
 			this.logger.trace("No such ACS node: {}; skipping ...", nodeDbId);
@@ -250,56 +276,86 @@ public class AcsReconcileService implements InitializingBean, DisposableBean {
 			return;
 		}
 		
-		if (!reindexUnreconciled) {
+		if (!index) {
 			this.logger.debug("A node in the DB is not indexed in Solr: {}: {}", nodeDbId, nodeRef);
 			this.reconcileLogger.info("UNRECONCILED: {} <=> {}", nodeDbId, nodeRef);
 			callback.unreconciled(nodeDbId);
 		} else {
-			logger.debug("A node in the DB is not indexed in Solr; attempt to reindex: {}: {}", nodeDbId, nodeRef);
-			this.reindex(nodeDbId, nodeRef, callback, execTimeout, execUnit);
+			this.logger.debug("A node in the DB is not indexed in Solr; attempt to index: {}: {}", nodeDbId, nodeRef);
+			this.index(nodeDbId, nodeRef, callback);
+			// purposefully forgetting about the returned future
+			// its results will be logged
+			// the reconcile thread will continue independently
+			// the callback will lag
 		}
 	}
 	
-	public void reindex(long nodeDbId, NodeRef nodeRef,
-			ReconcileCallback callback,
-			long execTimeout, TimeUnit execUnit) throws InterruptedException, TimeoutException {
-		Set<ShardInstance> syncHosts = new HashSet<>();
-		Set<ShardInstance> asyncHosts = new HashSet<>();
-		Map<ShardInstance, String> errorHosts = new HashMap<>();
-		
-		ReindexCallback reindexCallback = new ReindexCallback() {
+	public Future<Void> index(long nodeDbId, NodeRef nodeRef, ReconcileCallback callback) throws InterruptedException, TimeoutException {
+		IndexCallback indexCallback = new IndexCallback() {
 			
 			@Override
 			public void success(ShardInstance instance) {
-				reconcileLogger.info("REINDEXED: {} <=> {}", nodeDbId, nodeRef);
-				syncHosts.add(instance);
+				reconcileLogger.info("INDEXED: {} <=> {} in {}", nodeDbId, nodeRef, instance);
+				if (callback != null)
+					callback.processed(nodeDbId,
+						Collections.singleton(instance), Collections.emptySet(),
+						Collections.emptyMap());
 			}
 			
 			@Override
 			public void scheduled(ShardInstance instance) {
-				reconcileLogger.info("REINDEXING: {} <=> {}", nodeDbId, nodeRef);
-				asyncHosts.add(instance);
+				reconcileLogger.info("INDEXING: {} <=> {} in {}", nodeDbId, nodeRef, instance);
+				if (callback != null)
+					callback.processed(nodeDbId,
+						Collections.emptySet(), Collections.singleton(instance),
+						Collections.emptyMap());
 			}
 			
 			@Override
 			public void error(ShardInstance instance, String message) {
-				reconcileLogger.info("UNINDEXED: {} <=> {}", nodeDbId, nodeRef);
-				errorHosts.put(instance, message);
+				reconcileLogger.info("FAILED INDEX: {} <=> {} in {}", nodeDbId, nodeRef, instance);
+				if (callback != null)
+					callback.processed(nodeDbId,
+						Collections.emptySet(), Collections.emptySet(),
+						Collections.singletonMap(instance, message));
 			}
 		};
 		
-		try {
-			if (execTimeout < 0L) {
-				this.reindexService.reindex(nodeDbId, reindexCallback).get();
-			} else {
-				this.reindexService.reindex(nodeDbId, reindexCallback).get(execTimeout, execUnit);
+		return this.indexService.index(nodeDbId, indexCallback);
+	}
+	
+	public Future<Void> reindex(long nodeDbId, NodeRef nodeRef, ReconcileCallback callback) throws InterruptedException {
+		ReindexCallback reindexCallback = new ReindexCallback() {
+			
+			@Override
+			public void success(ShardInstance instance) {
+				reconcileLogger.info("REINDEXED: {} <=> {} in {}", nodeDbId, nodeRef, instance);
+				if (callback != null)
+					callback.processed(nodeDbId,
+						Collections.singleton(instance), Collections.emptySet(),
+						Collections.emptyMap());
 			}
-		} catch (ExecutionException ee) {
-			throw new RuntimeException("An unexpected exception occurred: " + ee.getMessage(), ee);
-		}
-		
-		if (callback != null)
-			callback.processed(nodeDbId, syncHosts, asyncHosts, errorHosts);
+			
+			@Override
+			public void scheduled(ShardInstance instance) {
+				reconcileLogger.info("REINDEXING: {} <=> {} in {}", nodeDbId, nodeRef, instance);
+				if (callback != null)
+					callback.processed(nodeDbId,
+						Collections.emptySet(), Collections.singleton(instance),
+						Collections.emptyMap());
+			}
+			
+			@Override
+			public void error(ShardInstance instance, String message) {
+				reconcileLogger.info("FAILED REINDEX: {} <=> {} in {}", nodeDbId, nodeRef, instance);
+				if (callback != null)
+					callback.processed(nodeDbId,
+						Collections.emptySet(), Collections.emptySet(),
+						Collections.singletonMap(instance, message));
+			}
+		};
+
+		return this.reindexService.reindex(nodeDbId, reindexCallback);
 	}
 	
 	private String formatForFts(QName qname) {

shared/src/main/java/com/inteligr8/alfresco/asie/service/ApiService.java

@@ -43,7 +43,7 @@ public class ApiService implements InitializingBean {
 	@Value("${inteligr8.asie.basePath}")
 	private String solrBaseUrl;
 	
-	@Value("${inteligr8.asie.reconciliation.nodesChunkSize:250}")
+	@Value("${inteligr8.asie.reconciliation.nodesChunkSize}")
 	private int nodesChunkSize;
 	
 	@Override

shared/src/main/java/com/inteligr8/alfresco/asie/service/ExecutorManager.java

@@ -6,6 +6,8 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionHandler;
 import java.util.concurrent.TimeUnit;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
@@ -31,41 +33,35 @@ import com.inteligr8.alfresco.asie.util.ThrottledThreadPoolExecutor;
 @Component
 public class ExecutorManager implements InitializingBean, DisposableBean, RemovalListener<String, ExecutorService> {
 	
-	@Value("${inteligr8.asie.executors.expireTimeInMinutes:30}")
+	private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+	@Value("${inteligr8.asie.executors.expireTimeInMinutes}")
 	private int expireTimeInMinutes;
 	
-	private Cache<String, ExecutorService> refCache;
-	private Cache<String, ExecutorService> expiringCache;
+	private Cache<String, ThrottledThreadPoolExecutor> cache;
 	
 	@Override
 	public void afterPropertiesSet() throws Exception {
-		// a weak value happens when the executor is no longer referenced
-		// the possible references are by the caller temporarily using and the `expiringCache` (below; so it expired)
-		// this keeps the pool from being shutdown after it expires if the caller is still referencing it
-		// ultimately, if it is cached, it will be in this cache and MAY be in the `expiringCache`. 
-		this.refCache = CacheBuilder.newBuilder()
-				.initialCapacity(8)
-				.weakValues()
-				.removalListener(this)
-				.build();
-		
-		this.expiringCache = CacheBuilder.newBuilder()
+		this.cache = CacheBuilder.newBuilder()
 				.initialCapacity(8)
 				.expireAfterAccess(this.expireTimeInMinutes, TimeUnit.MINUTES)
+				.removalListener(this)
 				.build();
 	}
 	
 	@Override
 	public void destroy() throws Exception {
-		this.refCache.invalidateAll();
-		this.refCache.cleanUp();
-		this.expiringCache.invalidateAll();
-		this.expiringCache.cleanUp();
+		this.cache.invalidateAll();
+		this.cache.cleanUp();
 	}
 	
 	@Override
 	public void onRemoval(RemovalNotification<String, ExecutorService> notification) {
-		notification.getValue().shutdown();
+		this.logger.debug("Throttled thread pool removed/expired from cache: {}", notification.getKey());
+		if (!notification.getValue().isShutdown()) {
+			notification.getValue().shutdown();
+			this.logger.info("Throttled thread pool shut down: {}", notification.getKey());
+		}
 	}
 	
 	public ThrottledThreadPoolExecutor createThrottled(
@@ -88,9 +84,10 @@ public class ExecutorManager implements InitializingBean, DisposableBean, Remova
             final RejectedExecutionHandler rejectedExecutionHandler) {
 		try {
 			// if it is already cached, reuse the cache; otherwise create one
-			final ExecutorService executor = this.refCache.get(name, new Callable<ThrottledThreadPoolExecutor>() {
+			return this.cache.get(name, new Callable<ThrottledThreadPoolExecutor>() {
 				@Override
 				public ThrottledThreadPoolExecutor call() {
+					logger.info("Creating throttled thread pool: {}", name);
 					ThrottledThreadPoolExecutor executor = null;
 					if (rejectedExecutionHandler == null) {
 						executor = new ThrottledThreadPoolExecutor(coreThreadPoolSize, maximumThreadPoolSize, maximumQueueSize,
@@ -103,14 +100,9 @@ public class ExecutorManager implements InitializingBean, DisposableBean, Remova
 								rejectedExecutionHandler);
 					}
 					
+					logger.debug("Created throttled thread pool: {}; threads: {}; queue: {}", name, maximumThreadPoolSize, maximumQueueSize);
 					executor.prestartAllCoreThreads();
-					return executor;
-				}
-			});
-			
-			return (ThrottledThreadPoolExecutor) this.expiringCache.get(name, new Callable<ExecutorService>() {
-				@Override
-				public ExecutorService call() throws Exception {
+					logger.trace("Started {} core threads in thread pool: {}", coreThreadPoolSize, name);
 					return executor;
 				}
 			});
@@ -120,19 +112,7 @@ public class ExecutorManager implements InitializingBean, DisposableBean, Remova
 	}
 	
 	public ExecutorService get(String name) {
-		// grab from the expiring cache first, so we can 
-		ExecutorService executor = this.expiringCache.getIfPresent(name);
-		if (executor != null)
-			return executor;
-		
-		executor = this.refCache.getIfPresent(name);
-		if (executor == null)
-			return null;
-		
-		// the executor expired, but it was still referenced by the caller
-		// re-cache it
-		this.expiringCache.put(name, executor);
-		return executor;
+		return this.cache.getIfPresent(name);
 	}
 	
 }

shared/src/main/java/com/inteligr8/alfresco/asie/service/IndexService.java

@@ -0,0 +1,43 @@
+package com.inteligr8.alfresco.asie.service;
+
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.springframework.stereotype.Component;
+
+import com.inteligr8.alfresco.asie.api.CoreAdminApi;
+import com.inteligr8.alfresco.asie.model.ActionCoreResponse;
+import com.inteligr8.alfresco.asie.model.core.IndexRequest;
+import com.inteligr8.alfresco.asie.spi.ActionCallback;
+import com.inteligr8.solr.model.Action;
+
+@Component
+public class IndexService extends AbstractNodeActionService {
+	
+	@Override
+	protected String getActionName() {
+		return "index";
+	}
+	
+	@Override
+	protected String getThreadNamePrefix() {
+		return "solr-index";
+	}
+	
+	@Override
+	protected ActionCoreResponse<Action> execute(CoreAdminApi api, String core, long nodeDbId) {
+		IndexRequest apiRequest = new IndexRequest().withCore(core).withNodeId(nodeDbId);
+		return api.index(apiRequest);
+	}
+	
+	public Future<Void> index(long nodeDbId, ActionCallback callback) throws InterruptedException {
+		return super.action(nodeDbId, callback);
+	}
+	
+	public void index(long nodeDbId, ActionCallback callback, long fullQueueTimeout, TimeUnit fullQueueUnit,
+			long execTimeout, TimeUnit execUnit) throws TimeoutException, InterruptedException {
+		super.action(nodeDbId, callback, fullQueueTimeout, fullQueueUnit, execTimeout, execUnit);
+	}
+
+}

shared/src/main/java/com/inteligr8/alfresco/asie/service/SolrShardHashService.java

@@ -1,14 +1,32 @@
 package com.inteligr8.alfresco.asie.service;
 
 import java.nio.charset.Charset;
+import java.util.regex.Matcher;
 
+import org.alfresco.error.AlfrescoRuntimeException;
+import org.alfresco.service.cmr.repository.NodeRef;
+import org.alfresco.service.cmr.repository.NodeService;
+import org.alfresco.service.namespace.NamespaceService;
+import org.alfresco.service.namespace.QName;
 import org.apache.commons.codec.digest.MurmurHash3;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import com.inteligr8.alfresco.asie.model.ShardSet;
+
 @Component
 public class SolrShardHashService {
 	
 	private final Charset charset = Charset.forName("utf-8");
+	private final Logger logger = LoggerFactory.getLogger(this.getClass());
+	
+	@Autowired
+	private NamespaceService namespaceService;
+	
+	@Autowired
+	private NodeService nodeService;
 	
 	public int hash(Object obj, int shardCount) {
 		String str = obj.toString();
@@ -23,5 +41,53 @@ public class SolrShardHashService {
 		hash.add(bytes, 0, bytes.length);
 		return Math.abs(hash.end()) % shardCount;
 	}
+	
+	/**
+	 * @param shardset A ShardSet.
+	 * @param nodeDbId The numeric database ID of an ACS node.
+	 * @return A shard instance from 0 to 1 less than the number of shards; -1 if unable to hash.
+	 */
+	public int computeShardInstanceId(ShardSet shardset, long nodeDbId) {
+		NodeRef nodeRef = this.nodeService.getNodeRef(nodeDbId);
+		if (nodeRef == null)
+			throw new AlfrescoRuntimeException("The node " + nodeDbId + " does not exist");
+		return this.computeShardInstanceId(shardset, nodeRef);
+	}
+
+	/**
+	 * @param shardset A ShardSet.
+	 * @param nodeRef A reference to an ACS node.
+	 * @return A shard instance from 0 to 1 less than the number of shards; -1 if unable to hash.
+	 */
+	public int computeShardInstanceId(ShardSet shardset, NodeRef nodeRef) {
+		switch (shardset.getMethod()) {
+			case PROPERTY:
+				QName hashableProperty = QName.createQName(shardset.getPrefixedProperty(), this.namespaceService);
+				Object fullPropertyValue = this.nodeService.getProperty(nodeRef, hashableProperty);
+				if (fullPropertyValue == null) {
+					this.logger.debug("Unable to determine shard instance ID because property does not exist on node: {}: {}", nodeRef, hashableProperty);
+					return -1;
+				}
+				
+				this.logger.trace("Discovered node property for sharding: {} => {}", nodeRef, fullPropertyValue);
+				String hashableValue = fullPropertyValue.toString();
+				if (shardset.getRegex() != null) {
+					Matcher matcher = shardset.getRegex().matcher(hashableValue);
+					if (!matcher.find()) {
+						this.logger.debug("Unable to determine shard instance ID because regex pattern doesn't match the hashing property value: {}: {}: {}", nodeRef, shardset.getRegex(), hashableValue);
+						return -1;
+					}
+					hashableValue = matcher.group(1);
+					this.logger.trace("Extracted shardable value from node: {}: {} => {}", nodeRef, fullPropertyValue, hashableValue);
+				}
+				
+				int shardHash = this.hash(hashableValue, shardset.getShards().intValue());
+				this.logger.debug("Hash shardable value to shard instance ID: {}: {} => {}", nodeRef, hashableValue, shardHash);
+				return shardHash;
+			default:
+				this.logger.trace("Unable to determine shard instance ID due to shard method: {}", shardset.getMethod());
+				return -1;
+		}
+	}
 
 }

shared/src/main/java/com/inteligr8/alfresco/asie/spi/IndexCallback.java

@@ -0,0 +1,12 @@
+package com.inteligr8.alfresco.asie.spi;
+
+import com.inteligr8.alfresco.asie.model.ShardInstance;
+
+public interface IndexCallback extends ActionCallback {
+	
+	@Override
+	default void unknownResult(ShardInstance instance) {
+		throw new IllegalStateException();
+	}
+
+}

shared/src/main/java/com/inteligr8/alfresco/asie/util/CompositeFuture.java

@@ -79,7 +79,7 @@ public class CompositeFuture<T> implements Future<T> {
 		List<T> results = new ArrayList<>(this.futures.size());
 		for (Future<T> future : this.futures) {
 			if (future instanceof RunnableFuture<?>) {
-				this.logger.debug("Waiting {} ms since the start of the exectuion of the future to complete", unit.toMillis(timeout));
+				this.logger.debug("Waiting {} ms since the start of the execution of the future to complete", unit.toMillis(timeout));
 				results.add(((RunnableFuture<T>) future).get(timeout, unit));
 			} else {
 				long remainingTimeMillis = expireTimeMillis - System.currentTimeMillis();
@@ -124,6 +124,7 @@ public class CompositeFuture<T> implements Future<T> {
 		List<CompositeFuture<?>> cfutures = new LinkedList<>();
 		int removedCancelled = 0;
 		int removedDone = 0;
+		int remain = 0;
 		
 		Iterator<Future<T>> i = this.futures.iterator();
 		while (i.hasNext()) {
@@ -132,16 +133,20 @@ public class CompositeFuture<T> implements Future<T> {
 				if (includeCancelled) {
 					removedCancelled++;
 					i.remove();
+				} else {
+					remain++;
 				}
 			} else if (future.isDone()) {
 				removedDone++;
 				i.remove();
 			} else if (future instanceof CompositeFuture<?>) {
 				cfutures.add((CompositeFuture<?>) future);
+			} else {
+				remain++;
 			}
 		}
 		
-		this.logger.debug("Purged {} cancelled and {} completed futures", removedCancelled, removedDone);
+		this.logger.debug("Purged {} cancelled and {} completed futures; {} remain", removedCancelled, removedDone, remain);
 		
 		for (CompositeFuture<?> cfuture : cfutures)
 			cfuture.purge(includeCancelled);

shared/src/main/java/com/inteligr8/alfresco/asie/util/ThrottledThreadPoolExecutor.java

@@ -80,14 +80,11 @@ public class ThrottledThreadPoolExecutor extends ThreadPoolExecutor {
 	}
 	
 	private WaitableRunnable submit(WaitableRunnable runnable, long throttlingBlockTimeout, TimeUnit throttlingBlockUnit) throws InterruptedException, TimeoutException {
-		// if no core threads are running, the queue won't be monitored for runnables
-		this.prestartAllCoreThreads();
-
 		if (throttlingBlockTimeout < 0L) {
 			this.getQueue().put(runnable);
 		} else {
 			if (!this.getQueue().offer(runnable, throttlingBlockTimeout, throttlingBlockUnit))
-				throw new TimeoutException();
+				throw new TimeoutException("Timeout waiting for queue space for runnable");
 		}
 		
 		return runnable;

shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/indexAcsNode.post.desc.xml

@@ -0,0 +1,63 @@
+<webscript xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xsi:noNamespaceSchemaLocation="https://bitbucket.org/!api/2.0/snippets/inteligr8/AzMgbp/80fdd26a6b3769a63cdc6b54bf1f39e378545cf7/files/snippet.txt">
+	
+	<!-- Naming & Organization -->
+	<shortname>Index ACS Node in ASIE Indexes</shortname>
+	<family>Inteligr8 ASIE</family>
+	<description><![CDATA[
+		<p>Index the specified ACS node in the ASIE indexes.
+			This call will attempt to index the ACS node on all applicable Solr nodes.
+			The index operation could be synchronous or asynchronous and could fail on any Solr node.
+			If any Solr node failed synchronously in the execution, then expect a status code of 500.
+			The response body will still be identical to the 200/202 status codes.
+			If any Solr node is executing the index asynchronously and there are no synchronous failures, then expect a status code of 202.</p>
+		<p>The following path parameters are supported:</p>
+		<dl>
+			<dt>nodeId</dt>
+			<dd>An ACS node ID.</dd>
+		</dl>
+		<p>The following response body should be expected in most cases (200, 202, and 500 status codes):</p>
+		<pre>
+			{
+				"nodeDbId": number,
+				"success": [
+					"solrHostSync:8983/solr",
+					...
+				],
+				"scheduled": [
+					"solrHostAsync:8983/solr",
+					...
+				],
+				"error": [
+					"solrHostThatFailed:8983/solr": {
+						"message": "string"
+					},
+					...
+				]
+			}
+		</pre> 
+		<p>The following status codes should be expected:</p>
+		<dl>
+			<dt>200</dt>
+			<dd>OK</dd>
+			<dt>202</dt>
+			<dd>Accepted</dd>
+			<dt>400</dt>
+			<dd>The path or query parameters are invalid</dd>
+		</dl>
+	]]></description>
+	
+	<!-- Endpoint Configuration -->
+	<url>/inteligr8/asie/acs/node/{nodeId}/index</url>
+	<format default="json">any</format>
+	
+	<!-- Security -->
+	<authentication>user</authentication>
+	
+	<!-- Functionality -->
+	<cache>
+		<never>false</never>
+		<public>false</public>
+	</cache>
+	
+</webscript> 

shared/src/main/resources/alfresco/extension/templates/webscripts/com/inteligr8/alfresco/asie/reconcileAcsNodes.post.desc.xml

@@ -16,6 +16,13 @@
 			<dt>toDbId</dt>
 			<dd>A DB ID integer for the ending point of a range, exclusive.</dd>
 		</dl>
+		<p>The following query parameters are also supported:</p>
+		<dl>
+			<dt>index</dt>
+			<dd>Schedule an `INDEX` action against all unreconciled nodes.</dd>
+			<dt>reindex</dt>
+			<dd>Schedule a `REINDEX` action against all reconciled nodes.</dd>
+		</dl>
 		<p>The following response body should be expected in most cases (200, 202, and 500 status codes):</p>
 		<pre>
 			{
@@ -58,7 +65,7 @@
 	]]></description>
 	
 	<!-- Endpoint Configuration -->
-	<url>/inteligr8/asie/acs/nodes/{fromDbId}/{toDbId}/reconcile?reindex={reindex?}</url>
+	<url>/inteligr8/asie/acs/nodes/{fromDbId}/{toDbId}/reconcile?index={index?}&amp;reindex={reindex?}</url>
 	<format default="json">any</format>
 	
 	<!-- Security -->

shared/src/main/resources/alfresco/module/com_inteligr8_alfresco_asie-shared/alfresco-global.properties

@@ -1,8 +1,25 @@
 
 # defaulting to 3 days = 60 * 24 * 3 = 4320
+# once the node is selected, no other node for the shard will be used for the backup
 inteligr8.asie.backup.persistTimeMinutes=4320
 
+# what authorities (users or groups) may use the REST services provided by this module?
 inteligr8.asie.allowedAuthorities=GROUP_ALFRESCO_ADMINISTRATORS
 
 # same as solr.baseUrl, but that property is private to the Search subsystem
 inteligr8.asie.basePath=/solr
+
+
+# How long should idle executors remain before being shutdown?
+# They will re-initialize if needed again
+inteligr8.asie.executors.expireTimeInMinutes=30
+
+# Reconciliation configuration; each node will be processed in its own thread
+inteligr8.asie.reconciliation.nodesChunkSize=250
+inteligr8.asie.reconciliation.nodeTimeoutSeconds=10
+inteligr8.asie.reconciliation.concurrentQueueSize=32
+inteligr8.asie.reconciliation.concurrency=2
+
+# Action (like indexing and re-indexing) configuration
+inteligr8.asie.default.concurrentQueueSize=32
+inteligr8.asie.default.concurrency=2

solr-api/pom.xml

@@ -6,7 +6,7 @@
 	<parent>
 		<groupId>com.inteligr8.alfresco</groupId>
 		<artifactId>asie-platform-module-parent</artifactId>
-		<version>1.3.0</version>
+		<version>1.3.5</version>
 		<relativePath>../</relativePath>
 	</parent>