inteligr8/asie-platform-module
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved unload from shared to enterprise

Browse Source
This commit is contained in:
Brian Long
2024-11-06 13:22:52 -05:00
parent 57293c6efe
commit 20d9ce299a
6 changed files with 27 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/rest/AbstractUnregisterNodeWebScript.java Executable file
View File

@@ -0,0 +1,150 @@
package com.inteligr8.alfresco.asie.enterprise.rest;
import java.io.IOException;
import java.io.Serializable;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.alfresco.repo.index.shard.ShardState;
import org.alfresco.service.cmr.attributes.AttributeService;
import org.alfresco.service.cmr.attributes.AttributeService.AttributeQueryCallback;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.extensions.webscripts.WebScriptRequest;
import org.springframework.extensions.webscripts.WebScriptResponse;
import org.springframework.http.HttpStatus;
import com.inteligr8.alfresco.asie.Constants;
import com.inteligr8.alfresco.asie.api.CoreAdminApi;
import com.inteligr8.alfresco.asie.model.NodeParameterSet;
import com.inteligr8.alfresco.asie.rest.AbstractAsieNodeWebScript;
import com.inteligr8.alfresco.asie.service.ShardBackupService;
import com.inteligr8.alfresco.asie.spi.ShardStateService;
import com.inteligr8.solr.model.CoreMetadata;
import com.inteligr8.solr.model.core.StatusRequest;
import com.inteligr8.solr.model.core.StatusResponse;
import com.inteligr8.solr.model.core.UnloadRequest;
public abstract class AbstractUnregisterNodeWebScript<T extends NodeParameterSet> extends AbstractAsieNodeWebScript {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired
@Qualifier(Constants.QUALIFIER_ASIE)
private AttributeService attrService;
@Autowired
private ShardBackupService sbs;
@Autowired
private ShardStateService sss;
protected abstract T createParameters(WebScriptRequest req, String nodeHostname, int nodePort);
@Override
protected void execute(WebScriptRequest req, WebScriptResponse res, String nodeHostname, int nodePort)
throws IOException {
T params = this.createParameters(req, nodeHostname, nodePort);
final List<Pair<Serializable[], ShardState>> matchingCores = new LinkedList<>();
AttributeQueryCallback callback = new AttributeQueryCallback() {
@Override
public boolean handleAttribute(Long id, Serializable value, Serializable[] keys) {
ShardState shardState = (ShardState) value;
if (!matches(params, shardState))
return true;
matchingCores.add(Pair.of(keys, shardState));
return true;
}
};
this.sss.iterate(callback);
Serializable[] keys = new String[] {
Constants.ATTR_ASIE,
Constants.ATTR_UNLOADED,
nodeHostname + ":" + nodePort
};
@SuppressWarnings("unchecked")
Map<String, String> cores = (Map<String, String>) this.attrService.getAttribute(keys);
if (cores == null)
cores = new HashMap<>();
try {
for (Pair<Serializable[], ShardState> matchingCore : matchingCores) {
ShardState shardNode = matchingCore.getValue();
String core = shardNode.getPropertyBag().get("coreName");
try {
StatusResponse status = this.getCoreStatus(nodeHostname, nodePort, core);
if (status == null) {
this.logger.warn("Registered host/core status could not be retrieved: {}:{}/solr/{}", nodeHostname, nodePort, core);
} else {
CoreMetadata coreMetadata = status.getStatus().getCores().get(core);
if (coreMetadata == null || coreMetadata.getName() == null) {
this.logger.warn("Registered core does not actually exist on the node host; could be a DNS issue: {}:{}/solr/{}", nodeHostname, nodePort, core);
} else {
this.unloadCore(nodeHostname, nodePort, core);
cores.put(core, coreMetadata.getInstancePath());
}
}
} finally {
this.sss.remove(matchingCore.getKey());
this.sbs.forget(shardNode);
}
}
} finally {
// FIXME maybe a separate tx?
this.attrService.setAttribute((Serializable) cores, keys);
}
res.setStatus(HttpStatus.OK.value());
}
protected boolean matches(T params, ShardState shardState) {
if (!params.getHostname().equalsIgnoreCase(shardState.getShardInstance().getHostName())) {
InetAddress nodeAddress = params.getAddress();
if (nodeAddress == null)
return false;
InetAddress shardNodeAddress = resolve(shardState.getShardInstance().getHostName());
if (!nodeAddress.equals(shardNodeAddress))
return false;
}
if (params.getPort() != shardState.getShardInstance().getPort())
return false;
return true;
}
private InetAddress resolve(String hostname) {
try {
return InetAddress.getByName(hostname);
} catch (UnknownHostException uhe) {
// suppress
return null;
}
}
protected StatusResponse getCoreStatus(String nodeHostname, int nodePort, String core) {
this.logger.debug("Retrieving status for core {} on ASIE node: {}", core, nodeHostname);
CoreAdminApi api = this.createApi(nodeHostname, nodePort);
return api.getStatus(new StatusRequest().withCore(core));
}
protected void unloadCore(String nodeHostname, int nodePort, String core) {
this.logger.info("Unloading core {} on ASIE node: {}", core, nodeHostname);
CoreAdminApi api = this.createApi(nodeHostname, nodePort);
api.unload(new UnloadRequest().withCore(core));
}
}

136
enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/rest/ReloadNodeShardWebScript.java Executable file
View File

@@ -0,0 +1,136 @@
package com.inteligr8.alfresco.asie.enterprise.rest;
import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.alfresco.service.cmr.attributes.AttributeService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.extensions.webscripts.WebScriptException;
import org.springframework.extensions.webscripts.WebScriptRequest;
import org.springframework.extensions.webscripts.WebScriptResponse;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.Constants;
import com.inteligr8.alfresco.asie.api.CoreAdminApi;
import com.inteligr8.alfresco.asie.rest.AbstractAsieNodeWebScript;
import com.inteligr8.solr.model.ExceptionResponse;
import com.inteligr8.solr.model.core.CreateRequest;
import com.inteligr8.solr.model.core.ReloadRequest;
import jakarta.ws.rs.BadRequestException;
import jakarta.ws.rs.InternalServerErrorException;
@Component(value = "webscript.com.inteligr8.alfresco.asie.nodeShard.post")
public class ReloadNodeShardWebScript extends AbstractAsieNodeWebScript {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final Pattern shardRangePattern = Pattern.compile("([0-9]+)-([0-9]+)");
@Autowired
@Qualifier(Constants.QUALIFIER_ASIE)
private AttributeService attrService;
@Override
protected void execute(WebScriptRequest req, WebScriptResponse res, final String nodeHostname, final int nodePort)
throws IOException {
String shardCore = this.getRequiredPathParameter(req, "shardCore");
int shardId = this.getRequiredPathParameter(req, "shardId", Integer.class);
String coreName = shardCore + "-" + shardId;
Serializable[] keys = new String[] {
Constants.ATTR_ASIE,
Constants.ATTR_UNLOADED,
nodeHostname + ":" + nodePort
};
Map<String, String> cores = this.fetchUnloadedCores(keys);
if (!cores.containsKey(coreName)) {
cores.putAll(this.fetchOtherCores(req));
}
boolean changed = false;
String coreInstancePath = cores.get(coreName);
if (coreInstancePath == null)
throw new WebScriptException(HttpStatus.NOT_FOUND.value(), "The specified node/shard could not be found or formulated");
this.logger.info("Reloading core {} on ASIE node: {}", coreName, nodeHostname);
CoreAdminApi api = this.createApi(nodeHostname, nodePort);
try {
api.create(new CreateRequest()
.withCore(coreName)
.withConfigDirectory(coreInstancePath));
} catch (BadRequestException bre) {
this.logger.warn("Core {} does not exist on ASIE node: {}; forgetting it", coreName, nodeHostname);
cores.remove(coreName);
changed = true;
} catch (InternalServerErrorException isee) {
ExceptionResponse response = isee.getResponse().readEntity(ExceptionResponse.class);
if (response.getError() == null || response.getError().getMessage() == null || !response.getError().getMessage().endsWith(" already exists."))
throw isee;
this.logger.warn("Core {} was already loaded on ASIE node: {}; reloading ...", coreName, nodeHostname);
api.reload(new ReloadRequest()
.withCore(coreName));
}
if (changed)
this.attrService.setAttribute((Serializable) cores, keys);
res.setStatus(HttpStatus.OK.value());
}
private Map<String, String> fetchUnloadedCores(Serializable[] keys) {
@SuppressWarnings("unchecked")
Map<String, String> cores = (Map<String, String>) this.attrService.getAttribute(keys);
if (cores == null)
cores = new HashMap<String, String>();
return cores;
}
private Map<String, String> fetchOtherCores(WebScriptRequest req) {
String coreName = this.getOptionalQueryParameter(req, "coreName");
if (coreName == null)
coreName = this.getOptionalQueryParameter(req, "core");
String shardRange = this.getOptionalQueryParameter(req, "shardRange");
Short shardCount = this.getOptionalQueryParameter(req, "shardCount", Short.class);
if (coreName == null || shardRange == null || shardCount == null)
return Collections.emptyMap();
String template = this.getOptionalQueryParameter(req, "template");
if (template == null)
template = "rerank";
Short nodeId = this.getOptionalQueryParameter(req, "nodeId", Short.class);
if (nodeId == null)
nodeId = 1;
Short nodeCount = this.getOptionalQueryParameter(req, "nodeCount", Short.class);
if (nodeCount == null)
nodeCount = 2;
String baseConfigDirectory = template + "--" + coreName + "--shards--" + shardCount + "-x-1--node--" + nodeId + "-of-" + nodeCount;
Matcher matcher = this.shardRangePattern.matcher(shardRange);
if (!matcher.find())
throw new WebScriptException(HttpStatus.BAD_REQUEST.value(), "The 'shardRange' query parameter value is not formatted as expected");
Map<String, String> cores = new HashMap<>();
short startShardId = Short.parseShort(matcher.group(1));
short endShardId = Short.parseShort(matcher.group(2));
for (short shardId = startShardId; shardId <= endShardId; shardId++) {
String shardConfigDirectory = baseConfigDirectory + "/" + coreName + "-" + shardId;
cores.put(coreName, shardConfigDirectory);
}
return cores;
}
}

138
enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/rest/ReloadNodeWebScript.java Executable file
View File

@@ -0,0 +1,138 @@
package com.inteligr8.alfresco.asie.enterprise.rest;
import java.io.IOException;
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.alfresco.service.cmr.attributes.AttributeService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.extensions.webscripts.WebScriptException;
import org.springframework.extensions.webscripts.WebScriptRequest;
import org.springframework.extensions.webscripts.WebScriptResponse;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.Constants;
import com.inteligr8.alfresco.asie.api.CoreAdminApi;
import com.inteligr8.alfresco.asie.rest.AbstractAsieNodeWebScript;
import com.inteligr8.solr.model.ExceptionResponse;
import com.inteligr8.solr.model.core.CreateRequest;
import com.inteligr8.solr.model.core.ReloadRequest;
import jakarta.ws.rs.BadRequestException;
import jakarta.ws.rs.InternalServerErrorException;
@Component(value = "webscript.com.inteligr8.alfresco.asie.node.post")
public class ReloadNodeWebScript extends AbstractAsieNodeWebScript {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final Pattern shardRangePattern = Pattern.compile("([0-9]+)-([0-9]+)");
@Autowired
@Qualifier(Constants.QUALIFIER_ASIE)
private AttributeService attrService;
@Override
protected void execute(WebScriptRequest req, WebScriptResponse res, final String nodeHostname, final int nodePort)
throws IOException {
Serializable[] keys = new String[] {
Constants.ATTR_ASIE,
Constants.ATTR_UNLOADED,
nodeHostname + ":" + nodePort
};
Map<String, String> cores = this.fetchUnloadedCores(keys);
cores.putAll(this.fetchOtherCores(req));
if (cores.isEmpty())
throw new WebScriptException(HttpStatus.NOT_FOUND.value(), "The specified node was not found or formulated");
boolean changed = false;
Iterator<Entry<String, String>> i = cores.entrySet().iterator();
while (i.hasNext()) {
Entry<String, String> core = i.next();
String coreName = core.getKey();
String coreInstancePath = core.getValue();
this.logger.info("Reloading core {} on ASIE node: {}", coreName, nodeHostname);
CoreAdminApi api = this.createApi(nodeHostname, nodePort);
try {
api.create(new CreateRequest()
.withCore(coreName)
.withConfigDirectory(coreInstancePath));
} catch (BadRequestException bre) {
this.logger.warn("Core {} does not exist on ASIE node: {}; forgetting it", coreName, nodeHostname);
i.remove();
changed = true;
} catch (InternalServerErrorException isee) {
ExceptionResponse response = isee.getResponse().readEntity(ExceptionResponse.class);
if (response.getError() == null || response.getError().getMessage() == null || !response.getError().getMessage().endsWith(" already exists."))
throw isee;
this.logger.info("Core {} was already loaded on ASIE node: {}; reloading ...", coreName, nodeHostname);
api.reload(new ReloadRequest()
.withCore(coreName));
}
}
if (changed)
this.attrService.setAttribute((Serializable) cores, keys);
res.setStatus(HttpStatus.OK.value());
}
private Map<String, String> fetchUnloadedCores(Serializable[] keys) {
@SuppressWarnings("unchecked")
Map<String, String> cores = (Map<String, String>) this.attrService.getAttribute(keys);
if (cores == null)
cores = new HashMap<String, String>();
return cores;
}
private Map<String, String> fetchOtherCores(WebScriptRequest req) {
String coreName = this.getOptionalQueryParameter(req, "coreName");
if (coreName == null)
coreName = this.getOptionalQueryParameter(req, "core");
String shardRange = this.getOptionalQueryParameter(req, "shardRange");
Short shardCount = this.getOptionalQueryParameter(req, "shardCount", Short.class);
if (coreName == null || shardRange == null || shardCount == null)
return Collections.emptyMap();
String template = this.getOptionalQueryParameter(req, "template");
if (template == null)
template = "rerank";
Short nodeId = this.getOptionalQueryParameter(req, "nodeId", Short.class);
if (nodeId == null)
nodeId = 1;
Short nodeCount = this.getOptionalQueryParameter(req, "nodeCount", Short.class);
if (nodeCount == null)
nodeCount = 2;
String baseConfigDirectory = template + "--" + coreName + "--shards--" + shardCount + "-x-1--node--" + nodeId + "-of-" + nodeCount;
Matcher matcher = this.shardRangePattern.matcher(shardRange);
if (!matcher.find())
throw new WebScriptException(HttpStatus.BAD_REQUEST.value(), "The 'shardRange' query parameter value is not formatted as expected");
Map<String, String> cores = new HashMap<>();
short startShardId = Short.parseShort(matcher.group(1));
short endShardId = Short.parseShort(matcher.group(2));
for (short shardId = startShardId; shardId <= endShardId; shardId++) {
String shardConfigDirectory = baseConfigDirectory + "/" + coreName + "-" + shardId;
cores.put(coreName, shardConfigDirectory);
}
return cores;
}
}

31
enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/rest/UnloadNodeShardWebScript.java Executable file
View File

@@ -0,0 +1,31 @@
package com.inteligr8.alfresco.asie.enterprise.rest;
import org.alfresco.repo.index.shard.ShardState;
import org.springframework.extensions.webscripts.WebScriptRequest;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.model.NodeShardParameterSet;
import com.inteligr8.alfresco.asie.model.ShardSet;
@Component(value = "webscript.com.inteligr8.alfresco.asie.nodeShard.delete")
public class UnloadNodeShardWebScript extends AbstractUnregisterNodeWebScript<NodeShardParameterSet> {
@Override
protected NodeShardParameterSet createParameters(WebScriptRequest req, String nodeHostname, int nodePort) {
ShardSet shardSet = this.getRequiredPathParameter(req, "shardSet", ShardSet.class);
int shardId = this.getRequiredPathParameter(req, "shardId", Integer.class);
return new NodeShardParameterSet(nodeHostname, nodePort, shardSet, shardId);
}
@Override
protected boolean matches(NodeShardParameterSet params, ShardState shardState) {
if (!params.getShardSet().isFor(shardState))
return false;
if (params.getShardId() != shardState.getShardInstance().getShard().getInstance())
return false;
return true;
}
}

16
enterprise-module/src/main/java/com/inteligr8/alfresco/asie/enterprise/rest/UnloadNodeWebScript.java Executable file
View File

@@ -0,0 +1,16 @@
package com.inteligr8.alfresco.asie.enterprise.rest;
import org.springframework.extensions.webscripts.WebScriptRequest;
import org.springframework.stereotype.Component;
import com.inteligr8.alfresco.asie.model.NodeParameterSet;
@Component(value = "webscript.com.inteligr8.alfresco.asie.node.delete")
public class UnloadNodeWebScript extends AbstractUnregisterNodeWebScript<NodeParameterSet> {
@Override
protected NodeParameterSet createParameters(WebScriptRequest req, String nodeHostname, int nodePort) {
return new NodeParameterSet(nodeHostname, nodePort);
}
}