diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index f714307dc..7255a8d65 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -19,58 +19,25 @@ package org.alfresco.solr; -import static java.util.Optional.ofNullable; - -import static org.alfresco.solr.HandlerOfResources.extractCustomProperties; -import static org.alfresco.solr.HandlerOfResources.getSafeBoolean; -import static org.alfresco.solr.HandlerOfResources.getSafeLong; -import static org.alfresco.solr.HandlerOfResources.openResource; -import static org.alfresco.solr.HandlerOfResources.updatePropertiesFile; -import static org.alfresco.solr.HandlerOfResources.updateSharedProperties; -import static org.alfresco.solr.HandlerReportBuilder.addCoreSummary; -import static org.alfresco.solr.HandlerReportBuilder.buildAclReport; -import static org.alfresco.solr.HandlerReportBuilder.buildAclTxReport; -import static org.alfresco.solr.HandlerReportBuilder.buildNodeReport; -import static org.alfresco.solr.HandlerReportBuilder.buildTrackerReport; -import static org.alfresco.solr.HandlerReportBuilder.buildTxReport; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - import com.google.common.collect.ImmutableMap; - import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.httpclient.AuthenticationException; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.SOLRAPIClientFactory; import org.alfresco.solr.config.ConfigUtil; import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.CoreStatePublisher; import org.alfresco.solr.tracker.DBIDRangeRouter; import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; +import org.alfresco.solr.utils.Utils; +import org.alfresco.util.Pair; import org.alfresco.util.shard.ExplicitShardingPolicy; -import org.apache.commons.codec.EncoderException; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.io.FileUtils; import org.apache.solr.common.SolrException; @@ -87,20 +54,84 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static java.util.Arrays.stream; +import static java.util.Optional.ofNullable; +import static org.alfresco.solr.HandlerOfResources.extractCustomProperties; +import static org.alfresco.solr.HandlerOfResources.getSafeBoolean; +import static org.alfresco.solr.HandlerOfResources.getSafeLong; +import static org.alfresco.solr.HandlerOfResources.openResource; +import static org.alfresco.solr.HandlerOfResources.updatePropertiesFile; +import static org.alfresco.solr.HandlerOfResources.updateSharedProperties; +import static org.alfresco.solr.HandlerReportHelper.addMasterOrStandaloneCoreSummary; +import static org.alfresco.solr.HandlerReportHelper.addSlaveCoreSummary; +import static org.alfresco.solr.HandlerReportHelper.buildAclReport; +import static org.alfresco.solr.HandlerReportHelper.buildAclTxReport; +import static org.alfresco.solr.HandlerReportHelper.buildNodeReport; +import static org.alfresco.solr.HandlerReportHelper.buildTrackerReport; +import static org.alfresco.solr.HandlerReportHelper.buildTxReport; +import static org.alfresco.solr.utils.Utils.notNullOrEmpty; + +/** + * Alfresco Solr administration endpoints provider. + * A customisation of the existing Solr {@link CoreAdminHandler} which offers additional administration endpoints. + * + * Since 1.5 the behaviour of these endpoints differs a bit depending on the target core. This because a lot of these + * endpoints rely on the information obtained from the trackers, and trackers (see SEARCH-1606) are disabled on slave + * cores. + * + * When a request arrives to this handler, the following are the possible scenarios: + * + * + * + * @author Andrea Gazzarini + */ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { protected static final Logger LOGGER = LoggerFactory.getLogger(AlfrescoCoreAdminHandler.class); - + + private static final String REPORT = "report"; private static final String ARG_ACLTXID = "acltxid"; - protected static final String ARG_TXID = "txid"; + static final String ARG_TXID = "txid"; private static final String ARG_ACLID = "aclid"; private static final String ARG_NODEID = "nodeid"; private static final String ARG_QUERY = "query"; - public static final String DATA_DIR_ROOT = "data.dir.root"; + private static final String DATA_DIR_ROOT = "data.dir.root"; public static final String ALFRESCO_DEFAULTS = "create.alfresco.defaults"; - public static final String NUM_SHARDS = "num.shards"; - public static final String SHARD_IDS = "shard.ids"; - public static final String DEFAULT_TEMPLATE = "rerank"; + private static final String NUM_SHARDS = "num.shards"; + private static final String SHARD_IDS = "shard.ids"; + static final String DEFAULT_TEMPLATE = "rerank"; static final String ALFRESCO_CORE_NAME = "alfresco"; static final String ARCHIVE_CORE_NAME = "archive"; @@ -112,8 +143,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private SolrTrackerScheduler scheduler; private TrackerRegistry trackerRegistry; - private ConcurrentHashMap informationServers = null; - + private ConcurrentHashMap informationServers; + + private static List CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index"); + public AlfrescoCoreAdminHandler() { super(); @@ -122,39 +155,23 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler public AlfrescoCoreAdminHandler(CoreContainer coreContainer) { super(coreContainer); - startup(coreContainer); - } - /** - * Startup services that exist outside of the core. - */ - public void startup(CoreContainer coreContainer) - { - LOGGER.info("Starting Alfresco core container services"); + LOGGER.info("Starting Alfresco Core Administration Services"); trackerRegistry = new TrackerRegistry(); informationServers = new ConcurrentHashMap<>(); this.scheduler = new SolrTrackerScheduler(this); String createDefaultCores = ConfigUtil.locateProperty(ALFRESCO_DEFAULTS, ""); - int numShards = Integer.valueOf(ConfigUtil.locateProperty(NUM_SHARDS, "1")); + int numShards = Integer.parseInt(ConfigUtil.locateProperty(NUM_SHARDS, "1")); String shardIds = ConfigUtil.locateProperty(SHARD_IDS, null); if (createDefaultCores != null && !createDefaultCores.isEmpty()) { - Runnable runnable = () -> + Thread thread = new Thread(() -> { - try - { - TimeUnit.SECONDS.sleep(10); //Wait a little for the container to start up - } - catch (InterruptedException e) - { - //Don't care - } + waitForTenSeconds(); setupNewDefaultCores(createDefaultCores, numShards, 1, 1, 1, shardIds); - }; - - Thread thread = new Thread(runnable); + }); thread.start(); } } @@ -179,7 +196,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @param numNodes - Not sure why the core needs to know this. * @param shardIds A comma separated list of shard ids for this core (or null). */ - void setupNewDefaultCores(String names, int numShards, int replicationFactor, int nodeInstance, int numNodes, String shardIds) + private void setupNewDefaultCores(String names, int numShards, int replicationFactor, int nodeInstance, int numNodes, String shardIds) { try { @@ -195,7 +212,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .filter(coreName -> !coreName.isEmpty()) .forEach(coreName -> { LOGGER.info("Attempting to create default alfresco core: {}", coreName); - if (!STORE_REF_MAP.keySet().contains(coreName)) + if (!STORE_REF_MAP.containsKey(coreName)) { throw new AlfrescoRuntimeException("Invalid '" + ALFRESCO_DEFAULTS + "' permitted values are " + STORE_REF_MAP.keySet()); } @@ -213,34 +230,34 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler /** * Shut down services that exist outside of the core. */ - public void shutdown() + @Override + public void shutdown() { super.shutdown(); - try + try { LOGGER.info("Shutting down Alfresco core container services"); + AlfrescoSolrDataModel.getInstance().close(); SOLRAPIClientFactory.close(); MultiThreadedHttpConnectionManager.shutdownAll(); - //Remove any core trackers still hanging around - trackerRegistry.getCoreNames().forEach(coreName -> trackerRegistry.removeTrackersForCore(coreName)); - - //Remove any information servers + coreNames().forEach(trackerRegistry::removeTrackersForCore); informationServers.clear(); - //Shutdown the scheduler and model tracker. if (!scheduler.isShutdown()) { scheduler.pauseAll(); + if (trackerRegistry.getModelTracker() != null) trackerRegistry.getModelTracker().shutdown(); + trackerRegistry.setModelTracker(null); scheduler.shutdown(); } - } - catch(Exception e) + } + catch(Exception exception) { - LOGGER.error("Problem shutting down", e); + LOGGER.error("Problem shutting down Alfresco core container services", exception); } } @@ -257,7 +274,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } catch (ClassNotFoundException e) { - return; + // Do nothing here } catch (Exception e) { @@ -267,117 +284,80 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler protected void handleCustomAction(SolrQueryRequest req, SolrQueryResponse rsp) { - LOGGER.info("######## Handle Custom Action ###########"); SolrParams params = req.getParams(); - String cname = params.get(CoreAdminParams.CORE); - String action = params.get(CoreAdminParams.ACTION); - action = action==null?"":action.toUpperCase(); + String action = + ofNullable(params.get(CoreAdminParams.ACTION)) + .map(String::trim) + .map(String::toUpperCase) + .orElse(""); try { - switch (action) { + switch (action) + { case "NEWCORE": + case "NEWINDEX": newCore(req, rsp); break; case "UPDATECORE": - updateCore(req, rsp); + case "UPDATEINDEX": + updateCore(req); break; case "UPDATESHARED": - updateShared(req, rsp); + updateShared(req); break; case "REMOVECORE": - removeCore(req, rsp); + removeCore(req); break; case "NEWDEFAULTINDEX": + case "NEWDEFAULTCORE": newDefaultCore(req, rsp); break; case "CHECK": - actionCHECK(cname); + actionCHECK(params); break; case "NODEREPORT": - actionNODEREPORTS(rsp, params, cname); + actionNODEREPORTS(rsp, params); break; case "ACLREPORT": - actionACLREPORT(rsp, params, cname); + actionACLREPORT(rsp, params); break; case "TXREPORT": - actionTXREPORT(rsp, params, cname); + actionTXREPORT(rsp, params); break; case "ACLTXREPORT": - actionACLTXREPORT(rsp, params, cname); + actionACLTXREPORT(rsp, params); break; case "RANGECHECK": - rangeCheck(rsp, cname); + rangeCheck(rsp, params); break; case "EXPAND": - expand(rsp, params, cname); + expand(rsp, params); break; case "REPORT": - actionREPORT(rsp, params, cname); + actionREPORT(rsp, params); break; case "PURGE": - if (cname != null) { - actionPURGE(params, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { - actionPURGE(params, coreName); - } - } + actionPURGE(params); break; case "REINDEX": - if (cname != null) { - actionREINDEX(params, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { - actionREINDEX(params, coreName); - } - } + actionREINDEX(params); break; case "RETRY": - if (cname != null) { - actionRETRY(rsp, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { - actionRETRY(rsp, coreName); - } - } + actionRETRY(rsp, params); break; case "INDEX": - if (cname != null) { - actionINDEX(params, cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { - actionINDEX(params, coreName); - } - } + actionINDEX(params); break; case "FIX": - if (cname != null) { - actionFIX(cname); - } else { - for (String coreName : getTrackerRegistry().getCoreNames()) { - actionFIX(coreName); - } - } + actionFIX(params); break; case "SUMMARY": - if (cname != null) { - NamedList report = new SimpleOrderedMap(); - actionSUMMARY(params, report, cname); - rsp.add("Summary", report); - } else { - NamedList report = new SimpleOrderedMap(); - for (String coreName : getTrackerRegistry().getCoreNames()) { - actionSUMMARY(params, report, coreName); - } - rsp.add("Summary", report); - } + actionSUMMARY(rsp, params); break; case "LOG4J": - String resource = "log4j-solr.properties"; - if (params.get("resource") != null) { - resource = params.get("resource"); - } - initResourceBasedLogging(resource); + initResourceBasedLogging( + ofNullable(params.get("resource")) + .orElse("log4j-solr.properties")); break; default: super.handleCustomAction(req, rsp); @@ -391,60 +371,61 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - private boolean newCore(SolrQueryRequest req, SolrQueryResponse rsp) { + private void newCore(SolrQueryRequest req, SolrQueryResponse rsp) + { SolrParams params = req.getParams(); req.getContext(); - // If numCore > 1 we are createing a collection of cores for a sole node in a cluster + // If numCore > 1 we are creating a collection of cores for a sole node in a cluster int numShards = params.getInt("numShards", 1); - String store = ""; - if (params.get("storeRef") != null) { - store = params.get("storeRef"); - } - if ((store == null) || (store.length() == 0)) { - return false; + String store = params.get("storeRef"); + if (store == null || store.trim().length() == 0) + { + return; } + StoreRef storeRef = new StoreRef(store); - String templateName = "vanilla"; - if (params.get("template") != null) { - templateName = params.get("template"); - } + String templateName = ofNullable(params.get("template")).orElse("vanilla"); int replicationFactor = params.getInt("replicationFactor", 1); int nodeInstance = params.getInt("nodeInstance", -1); int numNodes = params.getInt("numNodes", 1); - String coreName = params.get("coreName"); + String coreName = coreName(params); String shardIds = params.get("shardIds"); - Properties properties = extractCustomProperties(params); - return newCore(coreName, numShards, storeRef, templateName, replicationFactor, nodeInstance, numNodes, shardIds, properties, rsp); + newCore(coreName, numShards, storeRef, templateName, replicationFactor, nodeInstance, numNodes, shardIds, extractCustomProperties(params), rsp); } - private boolean newDefaultCore(SolrQueryRequest req, SolrQueryResponse rsp) { - + private void newDefaultCore(SolrQueryRequest req, SolrQueryResponse response) + { SolrParams params = req.getParams(); - String coreName = params.get("coreName") != null?params.get("coreName"):"alfresco"; - StoreRef storeRef = StoreRef.STORE_REF_WORKSPACE_SPACESSTORE; - String templateName = params.get("template") != null?params.get("template"): DEFAULT_TEMPLATE; + String coreName = ofNullable(coreName(params)).orElse(ALFRESCO_CORE_NAME); + String templateName = + params.get("template") != null + ? params.get("template") + : DEFAULT_TEMPLATE; + Properties extraProperties = extractCustomProperties(params); - if (params.get("storeRef") != null) { - String store = params.get("storeRef"); - storeRef = new StoreRef(store); - } - - return newDefaultCore(coreName, storeRef, templateName, extraProperties, rsp); + newDefaultCore( + coreName, + ofNullable(params.get("storeRef")) + .map(StoreRef::new) + .orElse(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE), + templateName, + extraProperties, + response); } - protected boolean newDefaultCore(String coreName, StoreRef storeRef, String templateName, Properties extraProperties, SolrQueryResponse rsp) + private void newDefaultCore(String coreName, StoreRef storeRef, String templateName, Properties extraProperties, SolrQueryResponse rsp) { - return newCore(coreName, 1, storeRef, templateName, 1, 1, 1, null, extraProperties, rsp); + newCore(coreName, 1, storeRef, templateName, 1, 1, 1, null, extraProperties, rsp); } - protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) + protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) { try { @@ -453,9 +434,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler File templates = new File(solrHome, "templates"); File template = new File(templates, templateName); - if(numShards > 1 ) + if(numShards > 1) { - String collectionName = templateName + "--" + storeRef.getProtocol() + "-" + storeRef.getIdentifier() + "--shards--"+numShards + "-x-"+replicationFactor+"--node--"+nodeInstance+"-of-"+numNodes; String coreBase = storeRef.getProtocol() + "-" + storeRef.getIdentifier() + "-"; if (coreName != null) @@ -463,14 +443,14 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler collectionName = templateName + "--" + coreName + "--shards--"+numShards + "-x-"+replicationFactor+"--node--"+nodeInstance+"-of-"+numNodes; coreBase = coreName + "-"; } - - File baseDirectory = new File(solrHome, collectionName); - + + File baseDirectory = new File(solrHome, collectionName); + if(nodeInstance == -1) { - return false; + return; } - + List shards; if(shardIds != null) { @@ -481,31 +461,29 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler ExplicitShardingPolicy policy = new ExplicitShardingPolicy(numShards, replicationFactor, numNodes); if(!policy.configurationIsValid()) { - return false; + return; } shards = policy.getShardIdsForNode(nodeInstance); } - + for(Integer shard : shards) { - coreName = coreBase+shard; + coreName = coreBase + shard; File newCore = new File(baseDirectory, coreName); String solrCoreName = coreName; if (coreName == null) { if(storeRef.equals(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE)) { - solrCoreName = "alfresco-"+shard; + solrCoreName = "alfresco-" + shard; } else if(storeRef.equals(StoreRef.STORE_REF_ARCHIVE_SPACESSTORE)) { - solrCoreName = "archive-"+shard; + solrCoreName = "archive-" + shard; } } createAndRegisterNewCore(rsp, extraProperties, storeRef, template, solrCoreName, newCore, numShards, shard, templateName); } - - return true; } else { @@ -515,54 +493,44 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } File newCore = new File(solrHome, coreName); createAndRegisterNewCore(rsp, extraProperties, storeRef, template, coreName, newCore, 0, 0, templateName); - - return true; } - } - catch (IOException e) + catch (IOException exception) { - e.printStackTrace(); - return false; + LOGGER.error("I/O Failure detected while creating the new core " + + "(name={}, numShard={}, storeRef={}, template={}, replication factor={}, node instance={}, num nodes={}, shard ids={})", + coreName, + numShards, + storeRef, + templateName, + replicationFactor, + nodeInstance, + numNodes, + shardIds, + exception); } } /** - * @param shardIds - * @return + * Extracts the list of shard identifiers from the given input string. + * the "excludeFromShardId" parameter is used to filter out those shards whose identifier is equal or greater than + * that parameter. + * + * @param shardIds the shards input string, where shards are separated by comma. + * @param excludeFromShardId filter out those shards whose identifier is equal or greater than this value. + * @return the list of shard identifiers. */ - private List extractShards(String shardIds, int numShards) + List extractShards(String shardIds, int excludeFromShardId) { - ArrayList shards = new ArrayList(); - for(String shardId : shardIds.split(",")) - { - try - { - Integer shard = Integer.valueOf(shardId); - if(shard.intValue() < numShards) - { - shards.add(shard); - } - } - catch(NumberFormatException nfe) - { - // ignore - } - } - return shards; + return stream(Objects.requireNonNullElse(shardIds, "").split(",")) + .map(String::trim) + .map(Utils::toIntOrNull) + .filter(Objects::nonNull) + .filter(shard -> shard < excludeFromShardId) + .collect(Collectors.toList()); } - /** - * @param rsp - * @param storeRef - * @param template - * @param coreName - * @param newCore - * @throws IOException - * @throws FileNotFoundException - */ - private void createAndRegisterNewCore(SolrQueryResponse rsp, Properties extraProperties, StoreRef storeRef, File template, String coreName, File newCore, int shardCount, int shardInstance, String templateName) throws IOException, - FileNotFoundException + private void createAndRegisterNewCore(SolrQueryResponse rsp, Properties extraProperties, StoreRef storeRef, File template, String coreName, File newCore, int shardCount, int shardInstance, String templateName) throws IOException { if (coreContainer.getLoadedCoreNames().contains(coreName)) { @@ -613,84 +581,60 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler properties.store(fileOutputStream, null); } - SolrCore core = coreContainer.create(coreName, newCore.toPath(), new HashMap(), false); + SolrCore core = coreContainer.create(coreName, newCore.toPath(), new HashMap<>(), false); rsp.add("core", core.getName()); } - /** - * Tests to see if one of the cores is an Alfresco special core! - * @param cores - * @return - */ - public boolean hasAlfrescoCore(Collection cores) + boolean hasAlfrescoCore(Collection cores) { - if (cores == null || cores.isEmpty()) return false; - for (SolrCore core:cores) - { - if (trackerRegistry.hasTrackersForCore(core.getName())) return true; - } - return false; + return notNullOrEmpty(cores).stream() + .map(SolrCore::getName) + .anyMatch(trackerRegistry::hasTrackersForCore); } - private boolean updateShared(SolrQueryRequest req, SolrQueryResponse rsp) + private void updateShared(SolrQueryRequest req) { SolrParams params = req.getParams(); - try { - + try + { File config = new File(AlfrescoSolrDataModel.getResourceDirectory(), AlfrescoSolrDataModel.SHARED_PROPERTIES); updateSharedProperties(params, config, hasAlfrescoCore(coreContainer.getCores())); - coreContainer.getCores().forEach(aCore -> coreContainer.reload(aCore.getName())); - - return true; - } catch (IOException e) + coreContainer.getCores().stream() + .map(SolrCore::getName) + .forEach(coreContainer::reload); + } + catch (IOException e) { LOGGER.error("Failed to update Shared properties ", e); } - return false; } - private boolean updateCore(SolrQueryRequest req, SolrQueryResponse rsp) + private void updateCore(SolrQueryRequest req) { - String coreName = null; - SolrParams params = req.getParams(); + ofNullable(coreName(req.getParams())) + .map(String::trim) + .filter(coreName -> !coreName.isEmpty()) + .ifPresent(coreName -> { + try (SolrCore core = coreContainer.getCore(coreName)) + { - if (params.get("coreName") != null) - { - coreName = params.get("coreName"); - } - - if ((coreName == null) || (coreName.length() == 0)) { return false; } + if (core == null) + { + return; + } - SolrCore core = null; - try { - core = coreContainer.getCore(coreName); + String configLocaltion = core.getResourceLoader().getConfigDir(); + File config = new File(configLocaltion, "solrcore.properties"); + updatePropertiesFile(req.getParams(), config, null); - if(core == null) - { - return false; - } - - String configLocaltion = core.getResourceLoader().getConfigDir(); - File config = new File(configLocaltion, "solrcore.properties"); - updatePropertiesFile(params, config, null); - - coreContainer.reload(coreName); - - return true; - } - finally - { - //Decrement core open count - if(core != null) - { - core.close(); - } - } + coreContainer.reload(coreName); + } + }); } - private boolean removeCore(SolrQueryRequest req, SolrQueryResponse rsp) + private void removeCore(SolrQueryRequest req) { String store = ""; SolrParams params = req.getParams(); @@ -699,414 +643,515 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler store = params.get("storeRef"); } - if ((store == null) || (store.length() == 0)) { return false; } + if ((store == null) || (store.length() == 0)) { return; } StoreRef storeRef = new StoreRef(store); - String coreName = storeRef.getProtocol() + "-" + storeRef.getIdentifier(); - if (params.get("coreName") != null) - { - coreName = params.get("coreName"); - } - // remove core + String coreName = ofNullable(coreName(req.getParams())).orElse(storeRef.getProtocol() + "-" + storeRef.getIdentifier()); coreContainer.unload(coreName, true, true, true); - - return true; } - - - private void actionFIX(String coreName) throws AuthenticationException, IOException, JSONException, EncoderException + private void actionCHECK(SolrParams params) { - // Gets Metadata health and fixes any problems - MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null, null); - IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb(); - toReindex.or(indexHealthReport.getDuplicatedTxInIndex()); - toReindex.or(indexHealthReport.getMissingTxFromIndex()); - long current = -1; - // Goes through problems in the index - while ((current = toReindex.nextSetBit(current + 1)) != -1) + String cname = coreName(params); + coreNames().stream() + .filter(coreName -> cname == null || coreName.equals(cname)) + .map(trackerRegistry::getTrackersForCore) + .flatMap(Collection::stream) + .map(Tracker::getTrackerState) + .forEach(state -> state.setCheck(true)); + } + + private void actionNODEREPORTS(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + Long dbid = + ofNullable(params.get(ARG_NODEID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No dbid parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(trackerRegistry::hasTrackersForCore) + .map(coreName -> new Pair<>(coreName, coreStatePublisher(coreName))) + .filter(coreNameAndPublisher -> coreNameAndPublisher.getSecond() != null) + .forEach(coreNameAndPublisher -> + report.add( + coreNameAndPublisher.getFirst(), + buildNodeReport(coreNameAndPublisher.getSecond(), dbid))); + } + + private void actionACLREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + Long aclid = + ofNullable(params.get(ARG_ACLID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLID + " parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .map(coreName -> new Pair<>(coreName, trackerRegistry.getTrackerForCore(coreName, AclTracker.class))) + .filter(coreNameAndAclTracker -> coreNameAndAclTracker.getSecond() != null) + .forEach(coreNameAndAclTracker -> + report.add( + coreNameAndAclTracker.getFirst(), + buildAclReport(coreNameAndAclTracker.getSecond(), aclid))); + + if (report.size() == 0) { - metadataTracker.addTransactionToReindex(current); - } - - // Gets the Acl health and fixes any problems - AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - indexHealthReport = aclTracker.checkIndex(null, null, null, null); - toReindex = indexHealthReport.getAclTxInIndexButNotInDb(); - toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex()); - toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); - current = -1; - // Goes through the problems in the index - while ((current = toReindex.nextSetBit(current + 1)) != -1) - { - aclTracker.addAclChangeSetToReindex(current); + addAlertMessage(report); } } - private void actionCHECK(String cname) + private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException { - if (cname != null) + String coreName = + ofNullable(coreName(params)) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + CoreAdminParams.CORE + " parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + if (isMasterOrStandalone(coreName)) { - for (Tracker tracker : trackerRegistry.getTrackersForCore(cname)) + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + Long txid = + ofNullable(params.get(ARG_TXID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_TXID + " parameter set.")); + + report.add(coreName, buildTxReport(trackerRegistry, informationServers.get(coreName), coreName, tracker, txid)); + } + else + { + addAlertMessage(report); + } + } + + private void actionACLTXREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException + { + Long acltxid = + ofNullable(params.get(ARG_ACLTXID)) + .map(Long::valueOf) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + ARG_ACLTXID + " parameter set.")); + + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .map(coreName -> new Pair<>(coreName, trackerRegistry.getTrackerForCore(coreName, AclTracker.class))) + .filter(coreNameAndAclTracker -> coreNameAndAclTracker.getSecond() != null) + .forEach(coreNameAndAclTracker -> + report.add( + coreNameAndAclTracker.getFirst(), + buildAclTxReport( + trackerRegistry, + informationServers.get(coreNameAndAclTracker.getFirst()), + coreNameAndAclTracker.getFirst(), + coreNameAndAclTracker.getSecond(), + acltxid))); + + if (report.size() == 0) + { + addAlertMessage(report); + } + } + + private void rangeCheck(SolrQueryResponse rsp, SolrParams params) throws IOException + { + String coreName = + ofNullable(coreName(params)) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + CoreAdminParams.CORE + " parameter set.")); + + if (isMasterOrStandalone(coreName)) + { + InformationServer informationServer = informationServers.get(coreName); + + DocRouter docRouter = getDocRouter(coreName); + + if(docRouter instanceof DBIDRangeRouter) { - tracker.getTrackerState().setCheck(true); + DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; + + if(!dbidRangeRouter.getInitialized()) + { + rsp.add("expand", 0); + rsp.add("exception", "DBIDRangeRouter not initialized yet."); + return; + } + + long startRange = dbidRangeRouter.getStartRange(); + long endRange = dbidRangeRouter.getEndRange(); + + long maxNodeId = informationServer.maxNodeId(); + long minNodeId = informationServer.minNodeId(); + long nodeCount = informationServer.nodeCount(); + + long bestGuess = -1; // -1 means expansion cannot be done. Either because expansion + // has already happened or we're above safe range + + long range = endRange - startRange; // We want this many nodes on the server + + long midpoint = startRange + ((long) (range * .5)); + + long safe = startRange + ((long) (range * .75)); + + long offset = maxNodeId-startRange; + + double density = 0; + + if(offset > 0) + { + density = ((double)nodeCount) / ((double)offset); // This is how dense we are so far. + } + + if (!dbidRangeRouter.getExpanded()) + { + if(maxNodeId <= safe) + { + if (maxNodeId >= midpoint) + { + if(density >= 1 || density == 0) + { + //This is fully dense shard or an empty shard. + // If it does happen, no expand is required. + bestGuess=0; + } + else + { + double multiplier = 1/density; + bestGuess = (long)(range*multiplier)-range; // This is how much to add + } + } + else + { + bestGuess = 0; // We're below the midpoint so it's to early to make a guess. + } + } + } + + rsp.add("start", startRange); + rsp.add("end", endRange); + rsp.add("nodeCount", nodeCount); + rsp.add("minDbid", minNodeId); + rsp.add("maxDbid", maxNodeId); + rsp.add("density", Math.abs(density)); + rsp.add("expand", bestGuess); + rsp.add("expanded", dbidRangeRouter.getExpanded()); + } + else + { + rsp.add("expand", -1); + rsp.add("exception", "ERROR: Wrong document router type:"+docRouter.getClass().getSimpleName()); } } else { - for (String core : trackerRegistry.getCoreNames()) + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + addAlertMessage(report); + } + } + + private synchronized void expand(SolrQueryResponse rsp, SolrParams params) throws IOException + { + String coreName = + ofNullable(coreName(params)) + .orElseThrow(() -> new AlfrescoRuntimeException("No " + CoreAdminParams.CORE + " parameter set.")); + + if (isMasterOrStandalone(coreName)) + { + InformationServer informationServer = informationServers.get(coreName); + DocRouter docRouter = getDocRouter(coreName); + + if(docRouter instanceof DBIDRangeRouter) { - for (Tracker tracker : trackerRegistry.getTrackersForCore(core)) + long expansion = Long.parseLong(params.get("add")); + DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter)docRouter; + + if(!dbidRangeRouter.getInitialized()) { - tracker.getTrackerState().setCheck(true); + rsp.add("expand", -1); + rsp.add("exception", "DBIDRangeRouter not initialized yet."); + return; + } + + if(dbidRangeRouter.getExpanded()) + { + rsp.add("expand", -1); + rsp.add("exception", "dbid range has already been expanded."); + return; + } + + long currentEndRange = dbidRangeRouter.getEndRange(); + long startRange = dbidRangeRouter.getStartRange(); + long maxNodeId = informationServer.maxNodeId(); + + long range = currentEndRange - startRange; + long safe = startRange + ((long) (range * .75)); + + if(maxNodeId > safe) + { + rsp.add("expand", -1); + rsp.add("exception", "Expansion cannot occur if max DBID in the index is more then 75% of range."); + return; + } + + long newEndRange = expansion+dbidRangeRouter.getEndRange(); + try + { + informationServer.capIndex(newEndRange); + informationServer.hardCommit(); + dbidRangeRouter.setEndRange(newEndRange); + dbidRangeRouter.setExpanded(true); + assert newEndRange == dbidRangeRouter.getEndRange(); + rsp.add("expand", dbidRangeRouter.getEndRange()); + } + catch(Throwable t) + { + rsp.add("expand", -1); + rsp.add("exception", t.getMessage()); + LOGGER.error("exception expanding", t); } } - } - } - - private void actionACLREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException, - JSONException - { - if (params.get(ARG_ACLID) == null) - { - throw new AlfrescoRuntimeException("No aclid parameter set"); - } - - if (cname != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - NamedList report = new SimpleOrderedMap(); - AclTracker tracker = trackerRegistry.getTrackerForCore(cname, AclTracker.class); - report.add(cname, buildAclReport(tracker, aclid)); - rsp.add("report", report); + else + { + rsp.add("expand", -1); + rsp.add("exception", "Wrong document router type:" + docRouter.getClass().getSimpleName()); + } } else { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - report.add(coreName, buildAclReport(tracker, aclid)); - } - rsp.add("report", report); + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); + addAlertMessage(report); } } - private void actionTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws AuthenticationException, - IOException, JSONException, EncoderException + private void actionREPORT(SolrQueryResponse rsp, SolrParams params) throws JSONException { - if (params.get(ARG_TXID) == null) - { - throw new AlfrescoRuntimeException("No txid parameter set"); - } - if (cname == null) - { - throw new AlfrescoRuntimeException("No cname parameter set"); - } + NamedList report = new SimpleOrderedMap<>(); + rsp.add(REPORT, report); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class); - Long txid = Long.valueOf(params.get(ARG_TXID)); - NamedList report = new SimpleOrderedMap(); - report.add(cname, buildTxReport(getTrackerRegistry(), informationServers.get(cname), cname, tracker, txid)); - rsp.add("report", report); - } - - private void actionACLTXREPORT(SolrQueryResponse rsp, SolrParams params, String cname) - throws AuthenticationException, IOException, JSONException, EncoderException - { - if (params.get(ARG_ACLTXID) == null) - { - throw new AlfrescoRuntimeException("No acltxid parameter set"); - } - - if (cname != null) - { - AclTracker tracker = trackerRegistry.getTrackerForCore(cname, AclTracker.class); - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - NamedList report = new SimpleOrderedMap(); - report.add(cname, buildAclTxReport(getTrackerRegistry(), informationServers.get(cname), cname, tracker, acltxid)); - rsp.add("report", report); - } - else - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - report.add(coreName, buildAclTxReport(getTrackerRegistry(), informationServers.get(coreName), coreName, tracker, acltxid)); - } - rsp.add("report", report); - } - } - - private void actionREPORT(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException, - JSONException, AuthenticationException, EncoderException - { Long fromTime = getSafeLong(params, "fromTime"); Long toTime = getSafeLong(params, "toTime"); Long fromTx = getSafeLong(params, "fromTx"); Long toTx = getSafeLong(params, "toTx"); Long fromAclTx = getSafeLong(params, "fromAclTx"); Long toAclTx = getSafeLong(params, "toAclTx"); - - if (cname != null) + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(trackerRegistry::hasTrackersForCore) + .filter(this::isMasterOrStandalone) + .forEach(coreName -> + report.add( + coreName, + buildTrackerReport( + trackerRegistry, + informationServers.get(coreName), + coreName, + fromTx, + toTx, + fromAclTx, + toAclTx, + fromTime, + toTime))); + + if (report.size() == 0) { - NamedList report = new SimpleOrderedMap(); - if (trackerRegistry.hasTrackersForCore(cname)) - { - report.add(cname, buildTrackerReport(getTrackerRegistry(), informationServers.get(cname),cname, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime)); - rsp.add("report", report); - } - else - { - report.add(cname, "Core unknown"); - } - } - else - { - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) - { - if (trackerRegistry.hasTrackersForCore(coreName)) - { - report.add(coreName, buildTrackerReport(getTrackerRegistry(), informationServers.get(coreName), coreName, fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime)); - } - else - { - report.add(coreName, "Core unknown"); - } - } - rsp.add("report", report); + addAlertMessage(report); } } - private DocRouter getDocRouter(String cname) + private void actionPURGE(SolrParams params) { - Collection trackers = trackerRegistry.getTrackersForCore(cname); - MetadataTracker metadataTracker = null; - for(Tracker tracker : trackers) - { - if(tracker instanceof MetadataTracker) - { - metadataTracker = (MetadataTracker)tracker; - } - } + Consumer purgeOnSpecificCore = coreName -> { + final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - DocRouter docRouter = metadataTracker.getDocRouter(); - return docRouter; + apply(params, ARG_TXID, metadataTracker::addTransactionToPurge); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToPurge); + apply(params, ARG_NODEID, metadataTracker::addNodeToPurge); + apply(params, ARG_ACLID, aclTracker::addAclToPurge); + }; + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(this::isMasterOrStandalone) + .forEach(purgeOnSpecificCore); } - - private void rangeCheck(SolrQueryResponse rsp,String cname) throws IOException + private void actionREINDEX(SolrParams params) { - InformationServer informationServer = informationServers.get(cname); + Consumer reindexOnSpecificCore = coreName -> { + final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - DocRouter docRouter = getDocRouter(cname); + apply(params, ARG_TXID, metadataTracker::addTransactionToReindex); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToReindex); + apply(params, ARG_NODEID, metadataTracker::addNodeToReindex); + apply(params, ARG_ACLID, aclTracker::addAclToReindex); - if(docRouter instanceof DBIDRangeRouter) { + ofNullable(params.get(ARG_QUERY)).ifPresent(metadataTracker::addQueryToReindex); + }; - DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; + String requestedCoreName = coreName(params); - if(!dbidRangeRouter.getInitialized()) - { - rsp.add("expand", 0); - rsp.add("exception", "DBIDRangeRouter not initialized yet."); - return; - } - - long startRange = dbidRangeRouter.getStartRange(); - long endRange = dbidRangeRouter.getEndRange(); - - long maxNodeId = informationServer.maxNodeId(); - long minNodeId = informationServer.minNodeId(); - long nodeCount = informationServer.nodeCount(); - - long bestGuess = -1; // -1 means expansion cannot be done. Either because expansion - // has already happened or we're above safe range - - long range = endRange - startRange; // We want this many nodes on the server - - long midpoint = startRange + ((long) (range * .5)); - - long safe = startRange + ((long) (range * .75)); - - long offset = maxNodeId-startRange; - - double density = 0; - - if(offset > 0) { - density = ((double)nodeCount) / ((double)offset); // This is how dense we are so far. - } - - if (!dbidRangeRouter.getExpanded()) - { - if(maxNodeId <= safe) - { - if (maxNodeId >= midpoint) - { - if(density >= 1 || density == 0) - { - //This is fully dense shard or an empty shard. - // If it does happen, no expand is required. - bestGuess=0; - } - else - { - double multiplier = 1/density; - bestGuess = (long)(range*multiplier)-range; // This is how much to add - } - } - else - { - bestGuess = 0; // We're below the midpoint so it's to early to make a guess. - } - } - } - - rsp.add("start", startRange); - rsp.add("end", endRange); - rsp.add("nodeCount", nodeCount); - rsp.add("minDbid", minNodeId); - rsp.add("maxDbid", maxNodeId); - rsp.add("density", Math.abs(density)); - rsp.add("expand", bestGuess); - rsp.add("expanded", dbidRangeRouter.getExpanded()); - } else { - rsp.add("expand", -1); - rsp.add("exception", "ERROR: Wrong document router type:"+docRouter.getClass().getSimpleName()); - return; - } + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(this::isMasterOrStandalone) + .forEach(reindexOnSpecificCore); } - private synchronized void expand(SolrQueryResponse rsp, SolrParams params, String cname) - throws IOException + private void actionRETRY(SolrQueryResponse rsp, SolrParams params) { - InformationServer informationServer = informationServers.get(cname); - DocRouter docRouter = getDocRouter(cname); + final Consumer retryOnSpecificCore = coreName -> { + MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + InformationServer srv = informationServers.get(coreName); - if(docRouter instanceof DBIDRangeRouter) - { - long expansion = Long.parseLong(params.get("add")); - DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter)docRouter; - - if(!dbidRangeRouter.getInitialized()) - { - rsp.add("expand", -1); - rsp.add("exception", "DBIDRangeRouter not initialized yet."); - return; - } - - if(dbidRangeRouter.getExpanded()) - { - rsp.add("expand", -1); - rsp.add("exception", "dbid range has already been expanded."); - return; - } - - long currentEndRange = dbidRangeRouter.getEndRange(); - long startRange = dbidRangeRouter.getStartRange(); - long maxNodeId = informationServer.maxNodeId(); - - long range = currentEndRange-startRange; - long safe = startRange + ((long) (range * .75)); - - if(maxNodeId > safe) - { - rsp.add("expand", -1); - rsp.add("exception", "Expansion cannot occur if max DBID in the index is more then 75% of range."); - return; - } - - long newEndRange = expansion+dbidRangeRouter.getEndRange(); try { - informationServer.capIndex(newEndRange); - informationServer.hardCommit(); - dbidRangeRouter.setEndRange(newEndRange); - dbidRangeRouter.setExpanded(true); - assert newEndRange == dbidRangeRouter.getEndRange(); - rsp.add("expand", dbidRangeRouter.getEndRange()); - return; - } - catch(Throwable t) - { - rsp.add("expand", -1); - rsp.add("exception", t.getMessage()); - LOGGER.error("exception expanding", t); - return; - } - } - else - { - rsp.add("expand", -1); - rsp.add("exception", "Wrong document router type:"+docRouter.getClass().getSimpleName()); - return; - } - } - - private void actionNODEREPORTS(SolrQueryResponse rsp, SolrParams params, String cname) throws IOException, - JSONException - { - if (cname != null) - { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(cname, - MetadataTracker.class); - Long dbid = null; - if (params.get(ARG_NODEID) != null) - { - dbid = Long.valueOf(params.get(ARG_NODEID)); - NamedList report = new SimpleOrderedMap(); - report.add(cname, buildNodeReport(tracker, dbid)); - rsp.add("report", report); - - } - else - { - throw new AlfrescoRuntimeException("No dbid parameter set"); - } - } - else - { - Long dbid = null; - if (params.get(ARG_NODEID) != null) - { - dbid = Long.valueOf(params.get(ARG_NODEID)); - NamedList report = new SimpleOrderedMap(); - for (String coreName : trackerRegistry.getCoreNames()) + for (Long nodeid : srv.getErrorDocIds()) { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, - MetadataTracker.class); - report.add(coreName, buildNodeReport(tracker, dbid)); + tracker.addNodeToReindex(nodeid); } - rsp.add("report", report); + rsp.add(coreName, srv.getErrorDocIds()); } - else + catch (Exception exception) { - throw new AlfrescoRuntimeException("No dbid parameter set"); + LOGGER.error("I/O Exception while adding Node to reindex.", exception); + } + }; + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(this::isMasterOrStandalone) + .forEach(retryOnSpecificCore); + } + + private void actionINDEX(SolrParams params) + { + Consumer indexOnSpecificCore = coreName -> { + final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + + apply(params, ARG_TXID, metadataTracker::addTransactionToIndex); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToIndex); + apply(params, ARG_NODEID, metadataTracker::addNodeToIndex); + apply(params, ARG_ACLID, aclTracker::addAclToIndex); + }; + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(this::isMasterOrStandalone) + .forEach(indexOnSpecificCore); + } + + private void actionFIX(SolrParams params) throws JSONException + { + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(this::isMasterOrStandalone) + .forEach(this::fixOnSpecificCore); + } + + private void fixOnSpecificCore(String coreName) + { + try + { + // Gets Metadata health and fixes any problems + MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null, null); + IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb(); + toReindex.or(indexHealthReport.getDuplicatedTxInIndex()); + toReindex.or(indexHealthReport.getMissingTxFromIndex()); + long current = -1; + // Goes through problems in the index + while ((current = toReindex.nextSetBit(current + 1)) != -1) { + metadataTracker.addTransactionToReindex(current); } + // Gets the Acl health and fixes any problems + AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + indexHealthReport = aclTracker.checkIndex(null, null, null, null); + toReindex = indexHealthReport.getAclTxInIndexButNotInDb(); + toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex()); + toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); + current = -1; + // Goes through the problems in the index + while ((current = toReindex.nextSetBit(current + 1)) != -1) { + aclTracker.addAclChangeSetToReindex(current); + } + } + catch(Exception exception) + { + throw new AlfrescoRuntimeException("", exception); } } - private void actionSUMMARY(SolrParams params, NamedList report, String coreName) throws IOException + void actionSUMMARY(SolrQueryResponse rsp, SolrParams params) + { + NamedList report = new SimpleOrderedMap<>(); + rsp.add("Summary", report); + + String requestedCoreName = coreName(params); + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .forEach(coreName -> coreSummary(params, report, coreName)); + } + + private void coreSummary(SolrParams params, NamedList report, String coreName) { boolean detail = getSafeBoolean(params, "detail"); boolean hist = getSafeBoolean(params, "hist"); boolean values = getSafeBoolean(params, "values"); boolean reset = getSafeBoolean(params, "reset"); - + InformationServer srv = informationServers.get(coreName); if (srv != null) { - addCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); - - if (reset) + try { - srv.getTrackerStats().reset(); + if (isMasterOrStandalone(coreName)) + { + addMasterOrStandaloneCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); + + if (reset) + { + srv.getTrackerStats().reset(); + } + } else + { + addSlaveCoreSummary(trackerRegistry, coreName, detail, hist, values, srv, report); + } + } + catch(Exception exception) + { + throw new AlfrescoRuntimeException("", exception); } } else @@ -1115,107 +1160,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } - - private void actionINDEX(SolrParams params, String coreName) + DocRouter getDocRouter(String cname) { - if (params.get(ARG_TXID) != null) - { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToIndex(txid); - } - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToIndex(acltxid); - } - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToIndex(nodeid); - } - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToIndex(aclid); - } - } - - private void actionRETRY(SolrQueryResponse rsp, String coreName) throws IOException - { - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - InformationServer srv = informationServers.get(coreName); - Set errorDocIds = srv.getErrorDocIds(); - for (Long nodeid : errorDocIds) - { - tracker.addNodeToReindex(nodeid); - } - rsp.add(coreName, errorDocIds); - } - - private void actionREINDEX(SolrParams params, String coreName) - { - if (params.get(ARG_TXID) != null) - { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToReindex(txid); - } - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToReindex(acltxid); - } - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToReindex(nodeid); - } - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToReindex(aclid); - } - if (params.get(ARG_QUERY) != null) - { - String query = params.get(ARG_QUERY); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addQueryToReindex(query); - } - } - - private void actionPURGE(SolrParams params, String coreName) - { - if (params.get(ARG_TXID) != null) - { - Long txid = Long.valueOf(params.get(ARG_TXID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addTransactionToPurge(txid); - } - if (params.get(ARG_ACLTXID) != null) - { - Long acltxid = Long.valueOf(params.get(ARG_ACLTXID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclChangeSetToPurge(acltxid); - } - if (params.get(ARG_NODEID) != null) - { - Long nodeid = Long.valueOf(params.get(ARG_NODEID)); - MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - tracker.addNodeToPurge(nodeid); - } - if (params.get(ARG_ACLID) != null) - { - Long aclid = Long.valueOf(params.get(ARG_ACLID)); - AclTracker tracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - tracker.addAclToPurge(aclid); - } + return ofNullable(trackerRegistry.getTrackerForCore(cname, MetadataTracker.class)) + .map(MetadataTracker::getDocRouter) + .orElse(null); } public ConcurrentHashMap getInformationServers() @@ -1228,7 +1177,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler return trackerRegistry; } - protected void setTrackerRegistry(TrackerRegistry trackerRegistry) + void setTrackerRegistry(TrackerRegistry trackerRegistry) { this.trackerRegistry = trackerRegistry; } @@ -1237,4 +1186,84 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { return scheduler; } -} + + private void waitForTenSeconds() + { + try + { + TimeUnit.SECONDS.sleep(10); + } + catch (InterruptedException e) + { + //Don't care + } + } + + /** + * Returns, for the given core, the component which is in charge to publish the core state. + * + * @param coreName the owning core name. + * @return the component which is in charge to publish the core state. + */ + CoreStatePublisher coreStatePublisher(String coreName) + { + return ofNullable(trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class)) + .map(CoreStatePublisher.class::cast) + .orElse(trackerRegistry.getTrackerForCore(coreName, SlaveCoreStatePublisher.class)); + } + + /** + * Quickly checks if the given name is associated to a master or standalone core. + * + * @param coreName the core name. + * @return true if the name is associated with a master or standalone mode, false otherwise. + */ + boolean isMasterOrStandalone(String coreName) + { + return trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class) != null; + } + + /** + * Adds to the returned report an information message alerting the receiver that this core is a slave, + * and therefore the same request should be re-submited to the corresponding master. + * + * @param report the response report. + */ + private void addAlertMessage(NamedList report) + { + report.add( + "WARNING", + "The requested endpoint is not available on the slave. " + + "Please re-submit the same request to the corresponding Master"); + } + + Collection coreNames() + { + return notNullOrEmpty(trackerRegistry.getCoreNames()); + } + + private void apply(SolrParams params, String parameterName, Consumer executeSideEffectAction) + { + ofNullable(params.get(parameterName)) + .map(Long::valueOf) + .ifPresent(executeSideEffectAction); + } + + /** + * Returns the core name indicated in the request parameters. + * A first attempt is done in order to check if a standard {@link CoreAdminParams#CORE} parameter is in the request. + * If not, the alternative "coreName" parameter name is used. + * + * @param params the request parameters. + * @return the core name specified in the request, null if the parameter is not found. + */ + String coreName(SolrParams params) + { + return CORE_PARAMETER_NAMES.stream() + .map(params::get) + .filter(Objects::nonNull) + .map(String::trim) + .findFirst() + .orElse(null); + } +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java deleted file mode 100644 index 5ccd37ed8..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportBuilder.java +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr; - -import org.alfresco.httpclient.AuthenticationException; -import org.alfresco.service.cmr.repository.datatype.Duration; -import org.alfresco.solr.client.Node; -import org.alfresco.solr.tracker.*; -import org.alfresco.util.CachingDateFormat; -import org.apache.commons.codec.EncoderException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.json.JSONException; - -import java.io.IOException; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Methods taken from AlfrescoCoreAdminHandler that deal with building reports - */ -public class HandlerReportBuilder { - - /** - * Builds AclReport - * @param tracker - * @param aclid - * @return - * @throws IOException - * @throws JSONException - */ - public static NamedList buildAclReport(AclTracker tracker, Long aclid) throws IOException, JSONException - { - AclReport aclReport = tracker.checkAcl(aclid); - - NamedList nr = new SimpleOrderedMap(); - nr.add("Acl Id", aclReport.getAclId()); - nr.add("Acl doc in index", aclReport.getIndexAclDoc()); - if (aclReport.getIndexAclDoc() != null) - { - nr.add("Acl tx in Index", aclReport.getIndexAclTx()); - } - - return nr; - } - - /** - * Builds TxReport - * @param trackerRegistry - * @param srv - * @param coreName - * @param tracker - * @param txid - * @return - * @throws AuthenticationException - * @throws IOException - * @throws JSONException - * @throws EncoderException - */ - public static NamedList buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) - throws AuthenticationException, IOException, JSONException, EncoderException - { - NamedList nr = new SimpleOrderedMap(); - nr.add("TXID", txid); - nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, txid, txid, 0l, 0l, null, null)); - NamedList nodes = new SimpleOrderedMap(); - // add node reports .... - List dbNodes = tracker.getFullNodesForDbTransaction(txid); - for (Node node : dbNodes) - { - nodes.add("DBID " + node.getId(), buildNodeReport(tracker, node)); - } - - nr.add("txDbNodeCount", dbNodes.size()); - nr.add("nodes", nodes); - return nr; - } - - /** - * Builds AclTxReport - * @param trackerRegistry - * @param srv - * @param coreName - * @param tracker - * @param acltxid - * @return - * @throws AuthenticationException - * @throws IOException - * @throws JSONException - * @throws EncoderException - */ - public static NamedList buildAclTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, AclTracker tracker, Long acltxid) - throws AuthenticationException, IOException, JSONException, EncoderException - { - NamedList nr = new SimpleOrderedMap(); - nr.add("TXID", acltxid); - nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, 0l, 0l, acltxid, acltxid, null, null)); - NamedList nodes = new SimpleOrderedMap(); - // add node reports .... - List dbAclIds = tracker.getAclsForDbAclTransaction(acltxid); - for (Long aclid : dbAclIds) - { - nodes.add("ACLID " + aclid, buildAclReport(tracker, aclid)); - } - nr.add("aclTxDbAclCount", dbAclIds.size()); - nr.add("nodes", nodes); - return nr; - } - - /** - * Builds Node report - * @param tracker - * @param node - * @return - * @throws IOException - * @throws JSONException - */ - public static NamedList buildNodeReport(MetadataTracker tracker, Node node) throws IOException, JSONException - { - NodeReport nodeReport = tracker.checkNode(node); - - NamedList nr = new SimpleOrderedMap(); - nr.add("Node DBID", nodeReport.getDbid()); - nr.add("DB TX", nodeReport.getDbTx()); - nr.add("DB TX status", nodeReport.getDbNodeStatus().toString()); - if (nodeReport.getIndexLeafDoc() != null) - { - nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx()); - } - if (nodeReport.getIndexAuxDoc() != null) - { - nr.add("Aux tx in Index", nodeReport.getIndexAuxTx()); - } - nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount()); - return nr; - } - - /** - * Builds Node Report - * @param tracker - * @param dbid - * @return - * @throws IOException - * @throws JSONException - */ - public static NamedList buildNodeReport(MetadataTracker tracker, Long dbid) throws IOException, JSONException - { - NodeReport nodeReport = tracker.checkNode(dbid); - - NamedList nr = new SimpleOrderedMap(); - nr.add("Node DBID", nodeReport.getDbid()); - nr.add("DB TX", nodeReport.getDbTx()); - nr.add("DB TX status", nodeReport.getDbNodeStatus().toString()); - if (nodeReport.getIndexLeafDoc() != null) - { - nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx()); - } - if (nodeReport.getIndexAuxDoc() != null) - { - nr.add("Aux tx in Index", nodeReport.getIndexAuxTx()); - } - nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount()); - return nr; - } - - /** - * Builds Tracker report - * @param trackerRegistry - * @param srv - * @param coreName - * @param fromTx - * @param toTx - * @param fromAclTx - * @param toAclTx - * @param fromTime - * @param toTime - * @return - * @throws IOException - * @throws JSONException - * @throws AuthenticationException - * @throws EncoderException - */ - public static NamedList buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx, - Long fromTime, Long toTime) throws IOException, JSONException, AuthenticationException, EncoderException - { - // ACL - AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime); - NamedList ihr = new SimpleOrderedMap(); - ihr.add("Alfresco version", aclTracker.getAlfrescoVersion()); - ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); - ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() - .cardinality()); - if (aclReport.getDuplicatedAclTxInIndex().cardinality() > 0) - { - ihr.add("First duplicate acl tx", aclReport.getDuplicatedAclTxInIndex().nextSetBit(0L)); - } - ihr.add("Count of acl transactions in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() - .cardinality()); - if (aclReport.getAclTxInIndexButNotInDb().cardinality() > 0) - { - ihr.add("First acl transaction in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() - .nextSetBit(0L)); - } - ihr.add("Count of missing acl transactions from the Index", aclReport.getMissingAclTxFromIndex() - .cardinality()); - if (aclReport.getMissingAclTxFromIndex().cardinality() > 0) - { - ihr.add("First acl transaction missing from the Index", aclReport.getMissingAclTxFromIndex() - .nextSetBit(0L)); - } - ihr.add("Index acl transaction count", aclReport.getAclTransactionDocsInIndex()); - ihr.add("Index unique acl transaction count", aclReport.getAclTransactionDocsInIndex()); - TrackerState aclState = aclTracker.getTrackerState(); - ihr.add("Last indexed change set commit time", aclState.getLastIndexedChangeSetCommitTime()); - Date lastChangeSetDate = new Date(aclState.getLastIndexedChangeSetCommitTime()); - ihr.add("Last indexed change set commit date", CachingDateFormat.getDateFormat().format(lastChangeSetDate)); - ihr.add("Last changeset id before holes", aclState.getLastIndexedChangeSetIdBeforeHoles()); - - // Metadata - MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime); - ihr.add("DB transaction count", metaReport.getDbTransactionCount()); - ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex() - .cardinality()); - if (metaReport.getDuplicatedTxInIndex().cardinality() > 0) - { - ihr.add("First duplicate", metaReport.getDuplicatedTxInIndex().nextSetBit(0L)); - } - ihr.add("Count of transactions in the index but not the DB", metaReport.getTxInIndexButNotInDb() - .cardinality()); - if (metaReport.getTxInIndexButNotInDb().cardinality() > 0) - { - ihr.add("First transaction in the index but not the DB", metaReport.getTxInIndexButNotInDb() - .nextSetBit(0L)); - } - ihr.add("Count of missing transactions from the Index", metaReport.getMissingTxFromIndex().cardinality()); - if (metaReport.getMissingTxFromIndex().cardinality() > 0) - { - ihr.add("First transaction missing from the Index", metaReport.getMissingTxFromIndex() - .nextSetBit(0L)); - } - ihr.add("Index transaction count", metaReport.getTransactionDocsInIndex()); - ihr.add("Index unique transaction count", metaReport.getTransactionDocsInIndex()); - ihr.add("Index node count", metaReport.getLeafDocCountInIndex()); - ihr.add("Count of duplicate nodes in the index", metaReport.getDuplicatedLeafInIndex().cardinality()); - if (metaReport.getDuplicatedLeafInIndex().cardinality() > 0) - { - ihr.add("First duplicate node id in the index", metaReport.getDuplicatedLeafInIndex().nextSetBit(0L)); - } - ihr.add("Index error count", metaReport.getErrorDocCountInIndex()); - ihr.add("Count of duplicate error docs in the index", metaReport.getDuplicatedErrorInIndex() - .cardinality()); - if (metaReport.getDuplicatedErrorInIndex().cardinality() > 0) - { - ihr.add("First duplicate error in the index", SolrInformationServer.PREFIX_ERROR - + metaReport.getDuplicatedErrorInIndex().nextSetBit(0L)); - } - ihr.add("Index unindexed count", metaReport.getUnindexedDocCountInIndex()); - ihr.add("Count of duplicate unindexed docs in the index", metaReport.getDuplicatedUnindexedInIndex() - .cardinality()); - if (metaReport.getDuplicatedUnindexedInIndex().cardinality() > 0) - { - ihr.add("First duplicate unindexed in the index", - metaReport.getDuplicatedUnindexedInIndex().nextSetBit(0L)); - } - TrackerState metaState = metadataTracker.getTrackerState(); - ihr.add("Last indexed transaction commit time", metaState.getLastIndexedTxCommitTime()); - Date lastTxDate = new Date(metaState.getLastIndexedTxCommitTime()); - ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate)); - ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles()); - - srv.addFTSStatusCounts(ihr); - - return ihr; - } - - - /** - * Adds a core summary - * @param cname - * @param detail - * @param hist - * @param values - * @param srv - * @param report - * @throws IOException - */ - public static void addCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values, - InformationServer srv, NamedList report) throws IOException - { - NamedList coreSummary = new SimpleOrderedMap(); - coreSummary.addAll((SimpleOrderedMap) srv.getCoreStats()); - - MetadataTracker metaTrkr = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class); - TrackerState metadataTrkrState = metaTrkr.getTrackerState(); - long lastIndexTxCommitTime = metadataTrkrState.getLastIndexedTxCommitTime(); - - long lastIndexedTxId = metadataTrkrState.getLastIndexedTxId(); - long lastTxCommitTimeOnServer = metadataTrkrState.getLastTxCommitTimeOnServer(); - long lastTxIdOnServer = metadataTrkrState.getLastTxIdOnServer(); - Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime); - Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer); - long transactionsToDo = lastTxIdOnServer - lastIndexedTxId; - if (transactionsToDo < 0) - { - transactionsToDo = 0; - } - - AclTracker aclTrkr = trackerRegistry.getTrackerForCore(cname, AclTracker.class); - TrackerState aclTrkrState = aclTrkr.getTrackerState(); - long lastIndexChangeSetCommitTime = aclTrkrState.getLastIndexedChangeSetCommitTime(); - long lastIndexedChangeSetId = aclTrkrState.getLastIndexedChangeSetId(); - long lastChangeSetCommitTimeOnServer = aclTrkrState.getLastChangeSetCommitTimeOnServer(); - long lastChangeSetIdOnServer = aclTrkrState.getLastChangeSetIdOnServer(); - Date lastIndexChangeSetCommitDate = new Date(lastIndexChangeSetCommitTime); - Date lastChangeSetOnServerDate = new Date(lastChangeSetCommitTimeOnServer); - long changeSetsToDo = lastChangeSetIdOnServer - lastIndexedChangeSetId; - if (changeSetsToDo < 0) - { - changeSetsToDo = 0; - } - - long nodesToDo = 0; - long remainingTxTimeMillis = 0; - if (transactionsToDo > 0) - { - // We now use the elapsed time as seen by the single thread farming out metadata indexing - double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx(); - double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime(); - nodesToDo = (long)(transactionsToDo * meanDocsPerTx); - remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime); - } - Date now = new Date(); - Date end = new Date(now.getTime() + remainingTxTimeMillis); - Duration remainingTx = new Duration(now, end); - - long remainingChangeSetTimeMillis = 0; - if (changeSetsToDo > 0) - { - // We now use the elapsed time as seen by the single thread farming out alc indexing - double meanAclsPerChangeSet = srv.getTrackerStats().getMeanAclsPerChangeSet(); - double meanAclElapsedIndexTime = srv.getTrackerStats().getMeanAclElapsedIndexTime(); - remainingChangeSetTimeMillis = (long) (changeSetsToDo * meanAclsPerChangeSet * meanAclElapsedIndexTime); - } - now = new Date(); - end = new Date(now.getTime() + remainingChangeSetTimeMillis); - Duration remainingChangeSet = new Duration(now, end); - - NamedList ftsSummary = new SimpleOrderedMap(); - long remainingContentTimeMillis = 0; - srv.addFTSStatusCounts(ftsSummary); - long cleanCount = ((Long)ftsSummary.get("Node count with FTSStatus Clean")).longValue(); - long dirtyCount = ((Long)ftsSummary.get("Node count with FTSStatus Dirty")).longValue(); - long newCount = ((Long)ftsSummary.get("Node count with FTSStatus New")).longValue(); - long nodesInIndex = ((Long)coreSummary.get("Alfresco Nodes in Index")); - long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0;; - if (dirtyCount + newCount + contentYetToSee > 0) - { - // We now use the elapsed time as seen by the single thread farming out alc indexing - double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime(); - remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime); - } - now = new Date(); - end = new Date(now.getTime() + remainingContentTimeMillis); - Duration remainingContent = new Duration(now, end); - coreSummary.add("FTS",ftsSummary); - - Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate); - if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0) - { - txLag = new Duration(); - } - long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000; - if (txLagSeconds < 0) - { - txLagSeconds = 0; - } - - Duration changeSetLag = new Duration(lastIndexChangeSetCommitDate, lastChangeSetOnServerDate); - if (lastIndexChangeSetCommitDate.compareTo(lastChangeSetOnServerDate) > 0) - { - changeSetLag = new Duration(); - } - long changeSetLagSeconds = (lastChangeSetCommitTimeOnServer - lastIndexChangeSetCommitTime) / 1000; - if (txLagSeconds < 0) - { - txLagSeconds = 0; - } - - ContentTracker contentTrkr = trackerRegistry.getTrackerForCore(cname, ContentTracker.class); - TrackerState contentTrkrState = contentTrkr.getTrackerState(); - // Leave ModelTracker out of this check, because it is common - boolean aTrackerIsRunning = aclTrkrState.isRunning() || metadataTrkrState.isRunning() - || contentTrkrState.isRunning(); - coreSummary.add("Active", aTrackerIsRunning); - - ModelTracker modelTrkr = trackerRegistry.getModelTracker(); - TrackerState modelTrkrState = modelTrkr.getTrackerState(); - coreSummary.add("ModelTracker Active", modelTrkrState.isRunning()); - coreSummary.add("ContentTracker Active", contentTrkrState.isRunning()); - coreSummary.add("MetadataTracker Active", metadataTrkrState.isRunning()); - coreSummary.add("AclTracker Active", aclTrkrState.isRunning()); - - // TX - - coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime); - coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate); - coreSummary.add("TX Lag", txLagSeconds + " s"); - coreSummary.add("TX Duration", txLag.toString()); - coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer); - coreSummary.add("Date for last TX on server", lastTxOnServerDate); - coreSummary.add("Id for last TX on server", lastTxIdOnServer); - coreSummary.add("Id for last TX in index", lastIndexedTxId); - coreSummary.add("Approx transactions remaining", transactionsToDo); - coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString()); - - // Change set - - coreSummary.add("Last Index Change Set Commit Time", lastIndexChangeSetCommitTime); - coreSummary.add("Last Index Change Set Commit Date", lastIndexChangeSetCommitDate); - coreSummary.add("Change Set Lag", changeSetLagSeconds + " s"); - coreSummary.add("Change Set Duration", changeSetLag.toString()); - coreSummary.add("Timestamp for last Change Set on server", lastChangeSetCommitTimeOnServer); - coreSummary.add("Date for last Change Set on server", lastChangeSetOnServerDate); - coreSummary.add("Id for last Change Set on server", lastChangeSetIdOnServer); - coreSummary.add("Id for last Change Set in index", lastIndexedChangeSetId); - coreSummary.add("Approx change sets remaining", changeSetsToDo); - coreSummary.add("Approx change set indexing time remaining", - remainingChangeSet.largestComponentformattedString()); - - coreSummary.add("Approx content indexing time remaining", - remainingContent.largestComponentformattedString()); - - // Stats - - coreSummary.add("Model sync times (ms)", - srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values)); - coreSummary.add("Acl index time (ms)", - srv.getTrackerStats().getAclTimes().getNamedList(detail, hist, values)); - coreSummary.add("Node index time (ms)", - srv.getTrackerStats().getNodeTimes().getNamedList(detail, hist, values)); - coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values)); - coreSummary.add("Doc Transformation time (ms)", srv.getTrackerStats().getDocTransformationTimes() - .getNamedList(detail, hist, values)); - - // Model - - Map> modelErrors = srv.getModelErrors(); - if (modelErrors.size() > 0) - { - NamedList errorList = new SimpleOrderedMap(); - for (Map.Entry> modelNameToErrors : modelErrors.entrySet()) - { - errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue()); - } - coreSummary.add("Model changes are not compatible with the existing data model and have not been applied", - errorList); - } - - report.add(cname, coreSummary); - } -} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java new file mode 100644 index 000000000..a3c2b069d --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java @@ -0,0 +1,564 @@ +/* + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.service.cmr.repository.datatype.Duration; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.tracker.*; +import org.alfresco.util.CachingDateFormat; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.json.JSONException; + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Optional.ofNullable; + +/** + * Methods taken from AlfrescoCoreAdminHandler that deal with building reports + */ +class HandlerReportHelper +{ + static NamedList buildAclReport(AclTracker tracker, Long aclid) throws JSONException + { + AclReport aclReport = tracker.checkAcl(aclid); + + NamedList nr = new SimpleOrderedMap<>(); + nr.add("Acl Id", aclReport.getAclId()); + nr.add("Acl doc in index", aclReport.getIndexAclDoc()); + if (aclReport.getIndexAclDoc() != null) + { + nr.add("Acl tx in Index", aclReport.getIndexAclTx()); + } + + return nr; + } + + static NamedList buildTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, MetadataTracker tracker, Long txid) throws JSONException + { + NamedList nr = new SimpleOrderedMap<>(); + nr.add("TXID", txid); + nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, txid, txid, 0L, 0L, null, null)); + NamedList nodes = new SimpleOrderedMap<>(); + + // add node reports .... + List dbNodes = tracker.getFullNodesForDbTransaction(txid); + for (Node node : dbNodes) + { + nodes.add("DBID " + node.getId(), buildNodeReport(tracker, node)); + } + + nr.add("txDbNodeCount", dbNodes.size()); + nr.add("nodes", nodes); + return nr; + } + + static NamedList buildAclTxReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, AclTracker tracker, Long acltxid) throws JSONException + { + try { + NamedList nr = new SimpleOrderedMap<>(); + nr.add("TXID", acltxid); + nr.add("transaction", buildTrackerReport(trackerRegistry, srv, coreName, 0L, 0L, acltxid, acltxid, null, null)); + NamedList nodes = new SimpleOrderedMap<>(); + + // add node reports .... + List dbAclIds = tracker.getAclsForDbAclTransaction(acltxid); + for (Long aclid : dbAclIds) { + nodes.add("ACLID " + aclid, buildAclReport(tracker, aclid)); + } + nr.add("aclTxDbAclCount", dbAclIds.size()); + nr.add("nodes", nodes); + return nr; + } + catch (Exception exception) + { + throw new AlfrescoRuntimeException("", exception); + } + } + + static NamedList buildNodeReport(MetadataTracker tracker, Node node) throws JSONException + { + NodeReport nodeReport = tracker.checkNode(node); + + NamedList nr = new SimpleOrderedMap<>(); + nr.add("Node DBID", nodeReport.getDbid()); + nr.add("DB TX", nodeReport.getDbTx()); + nr.add("DB TX status", nodeReport.getDbNodeStatus().toString()); + if (nodeReport.getIndexLeafDoc() != null) + { + nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx()); + } + if (nodeReport.getIndexAuxDoc() != null) + { + nr.add("Aux tx in Index", nodeReport.getIndexAuxTx()); + } + nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount()); + return nr; + } + + static NamedList buildNodeReport(CoreStatePublisher publisher, Long dbid) throws JSONException + { + NodeReport nodeReport = publisher.checkNode(dbid); + + NamedList payload = new SimpleOrderedMap<>(); + payload.add("Node DBID", nodeReport.getDbid()); + + if (publisher.isOnMasterOrStandalone()) + { + ofNullable(nodeReport.getDbTx()).ifPresent(value -> payload.add("DB TX", value)); + ofNullable(nodeReport.getDbNodeStatus()).map(Object::toString).ifPresent(value -> payload.add("DB TX Status", value)); + ofNullable(nodeReport.getIndexLeafTx()).ifPresent(value -> payload.add("Leaf tx in Index", value)); + ofNullable(nodeReport.getIndexAuxDoc()).ifPresent(value -> payload.add("Aux tx in Index", value)); + } + else + { + payload.add("WARNING", "This response comes from a slave core and it contains minimal information about the node. " + + "Please consider to re-submit the same request to the corresponding Master, in order to get more information."); + } + + ofNullable(nodeReport.getIndexedNodeDocCount()).ifPresent(value -> payload.add("Indexed Node Doc Count", value)); + + return payload; + } + + /** + * Builds Tracker report + */ + static NamedList buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx, + Long fromTime, Long toTime) throws JSONException + { + try + { + // ACL + AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime); + NamedList ihr = new SimpleOrderedMap<>(); + ihr.add("Alfresco version", aclTracker.getAlfrescoVersion()); + ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); + ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() + .cardinality()); + if (aclReport.getDuplicatedAclTxInIndex().cardinality() > 0) { + ihr.add("First duplicate acl tx", aclReport.getDuplicatedAclTxInIndex().nextSetBit(0L)); + } + ihr.add("Count of acl transactions in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() + .cardinality()); + if (aclReport.getAclTxInIndexButNotInDb().cardinality() > 0) { + ihr.add("First acl transaction in the index but not the DB", aclReport.getAclTxInIndexButNotInDb() + .nextSetBit(0L)); + } + ihr.add("Count of missing acl transactions from the Index", aclReport.getMissingAclTxFromIndex() + .cardinality()); + if (aclReport.getMissingAclTxFromIndex().cardinality() > 0) { + ihr.add("First acl transaction missing from the Index", aclReport.getMissingAclTxFromIndex() + .nextSetBit(0L)); + } + ihr.add("Index acl transaction count", aclReport.getAclTransactionDocsInIndex()); + ihr.add("Index unique acl transaction count", aclReport.getAclTransactionDocsInIndex()); + TrackerState aclState = aclTracker.getTrackerState(); + ihr.add("Last indexed change set commit time", aclState.getLastIndexedChangeSetCommitTime()); + Date lastChangeSetDate = new Date(aclState.getLastIndexedChangeSetCommitTime()); + ihr.add("Last indexed change set commit date", CachingDateFormat.getDateFormat().format(lastChangeSetDate)); + ihr.add("Last changeset id before holes", aclState.getLastIndexedChangeSetIdBeforeHoles()); + + // Metadata + MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime); + ihr.add("DB transaction count", metaReport.getDbTransactionCount()); + ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex() + .cardinality()); + if (metaReport.getDuplicatedTxInIndex().cardinality() > 0) { + ihr.add("First duplicate", metaReport.getDuplicatedTxInIndex().nextSetBit(0L)); + } + ihr.add("Count of transactions in the index but not the DB", metaReport.getTxInIndexButNotInDb() + .cardinality()); + if (metaReport.getTxInIndexButNotInDb().cardinality() > 0) { + ihr.add("First transaction in the index but not the DB", metaReport.getTxInIndexButNotInDb() + .nextSetBit(0L)); + } + ihr.add("Count of missing transactions from the Index", metaReport.getMissingTxFromIndex().cardinality()); + if (metaReport.getMissingTxFromIndex().cardinality() > 0) { + ihr.add("First transaction missing from the Index", metaReport.getMissingTxFromIndex() + .nextSetBit(0L)); + } + ihr.add("Index transaction count", metaReport.getTransactionDocsInIndex()); + ihr.add("Index unique transaction count", metaReport.getTransactionDocsInIndex()); + ihr.add("Index node count", metaReport.getLeafDocCountInIndex()); + ihr.add("Count of duplicate nodes in the index", metaReport.getDuplicatedLeafInIndex().cardinality()); + if (metaReport.getDuplicatedLeafInIndex().cardinality() > 0) { + ihr.add("First duplicate node id in the index", metaReport.getDuplicatedLeafInIndex().nextSetBit(0L)); + } + ihr.add("Index error count", metaReport.getErrorDocCountInIndex()); + ihr.add("Count of duplicate error docs in the index", metaReport.getDuplicatedErrorInIndex() + .cardinality()); + if (metaReport.getDuplicatedErrorInIndex().cardinality() > 0) { + ihr.add("First duplicate error in the index", SolrInformationServer.PREFIX_ERROR + + metaReport.getDuplicatedErrorInIndex().nextSetBit(0L)); + } + ihr.add("Index unindexed count", metaReport.getUnindexedDocCountInIndex()); + ihr.add("Count of duplicate unindexed docs in the index", metaReport.getDuplicatedUnindexedInIndex() + .cardinality()); + if (metaReport.getDuplicatedUnindexedInIndex().cardinality() > 0) { + ihr.add("First duplicate unindexed in the index", + metaReport.getDuplicatedUnindexedInIndex().nextSetBit(0L)); + } + TrackerState metaState = metadataTracker.getTrackerState(); + ihr.add("Last indexed transaction commit time", metaState.getLastIndexedTxCommitTime()); + Date lastTxDate = new Date(metaState.getLastIndexedTxCommitTime()); + ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate)); + ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles()); + + srv.addFTSStatusCounts(ihr); + + return ihr; + } + catch (Exception exception) + { + throw new AlfrescoRuntimeException("", exception); + } + } + + static void addSlaveCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values, + InformationServer srv, NamedList report) throws IOException + { + NamedList coreSummary = new SimpleOrderedMap<>(); + coreSummary.addAll((SimpleOrderedMap) srv.getCoreStats()); + + SlaveCoreStatePublisher statePublisher = trackerRegistry.getTrackerForCore(cname, SlaveCoreStatePublisher.class); + TrackerState trackerState = statePublisher.getTrackerState(); + long lastIndexTxCommitTime = trackerState.getLastIndexedTxCommitTime(); + + long lastIndexedTxId = trackerState.getLastIndexedTxId(); + long lastTxCommitTimeOnServer = trackerState.getLastTxCommitTimeOnServer(); + long lastTxIdOnServer = trackerState.getLastTxIdOnServer(); + + Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime); + Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer); + long transactionsToDo = lastTxIdOnServer - lastIndexedTxId; + if (transactionsToDo < 0) + { + transactionsToDo = 0; + } + + long nodesToDo = 0; + long remainingTxTimeMillis = 0; + if (transactionsToDo > 0) + { + // We now use the elapsed time as seen by the single thread farming out metadata indexing + double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx(); + double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime(); + nodesToDo = (long)(transactionsToDo * meanDocsPerTx); + remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime); + } + Date now = new Date(); + Date end = new Date(now.getTime() + remainingTxTimeMillis); + Duration remainingTx = new Duration(now, end); + + long remainingChangeSetTimeMillis = 0; + + now = new Date(); + end = new Date(now.getTime() + remainingChangeSetTimeMillis); + Duration remainingChangeSet = new Duration(now, end); + + NamedList ftsSummary = new SimpleOrderedMap<>(); + long remainingContentTimeMillis = 0; + srv.addFTSStatusCounts(ftsSummary); + long cleanCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Clean")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long dirtyCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Dirty")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long newCount = + ofNullable(ftsSummary.get("Node count with FTSStatus New")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long nodesInIndex = + ofNullable(coreSummary.get("Alfresco Nodes in Index")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0; + if (dirtyCount + newCount + contentYetToSee > 0) + { + // We now use the elapsed time as seen by the single thread farming out alc indexing + double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime(); + remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime); + } + now = new Date(); + end = new Date(now.getTime() + remainingContentTimeMillis); + Duration remainingContent = new Duration(now, end); + coreSummary.add("FTS",ftsSummary); + + Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate); + if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0) + { + txLag = new Duration(); + } + long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000; + if (txLagSeconds < 0) + { + txLagSeconds = 0; + } + + ModelTracker modelTrkr = trackerRegistry.getModelTracker(); + TrackerState modelTrkrState = modelTrkr.getTrackerState(); + coreSummary.add("ModelTracker Active", modelTrkrState.isRunning()); + coreSummary.add("NodeState Publisher Active", trackerState.isRunning()); + + // TX + + coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime); + coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate); + coreSummary.add("TX Lag", txLagSeconds + " s"); + coreSummary.add("TX Duration", txLag.toString()); + coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer); + coreSummary.add("Date for last TX on server", lastTxOnServerDate); + coreSummary.add("Id for last TX on server", lastTxIdOnServer); + coreSummary.add("Id for last TX in index", lastIndexedTxId); + coreSummary.add("Approx transactions remaining", transactionsToDo); + coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString()); + // Stats + + coreSummary.add("Model sync times (ms)", srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values)); + coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values)); + + // Model + + Map> modelErrors = srv.getModelErrors(); + if (modelErrors.size() > 0) + { + NamedList errorList = new SimpleOrderedMap<>(); + for (Map.Entry> modelNameToErrors : modelErrors.entrySet()) + { + errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue()); + } + coreSummary.add("Model changes are not compatible with the existing data model and have not been applied", errorList); + } + + report.add(cname, coreSummary); + } + + static void addMasterOrStandaloneCoreSummary(TrackerRegistry trackerRegistry, String cname, boolean detail, boolean hist, boolean values, + InformationServer srv, NamedList report) throws IOException + { + NamedList coreSummary = new SimpleOrderedMap<>(); + coreSummary.addAll((SimpleOrderedMap) srv.getCoreStats()); + + MetadataTracker metaTrkr = trackerRegistry.getTrackerForCore(cname, MetadataTracker.class); + TrackerState metadataTrkrState = metaTrkr.getTrackerState(); + long lastIndexTxCommitTime = metadataTrkrState.getLastIndexedTxCommitTime(); + + long lastIndexedTxId = metadataTrkrState.getLastIndexedTxId(); + long lastTxCommitTimeOnServer = metadataTrkrState.getLastTxCommitTimeOnServer(); + long lastTxIdOnServer = metadataTrkrState.getLastTxIdOnServer(); + Date lastIndexTxCommitDate = new Date(lastIndexTxCommitTime); + Date lastTxOnServerDate = new Date(lastTxCommitTimeOnServer); + long transactionsToDo = lastTxIdOnServer - lastIndexedTxId; + if (transactionsToDo < 0) + { + transactionsToDo = 0; + } + + AclTracker aclTrkr = trackerRegistry.getTrackerForCore(cname, AclTracker.class); + TrackerState aclTrkrState = aclTrkr.getTrackerState(); + long lastIndexChangeSetCommitTime = aclTrkrState.getLastIndexedChangeSetCommitTime(); + long lastIndexedChangeSetId = aclTrkrState.getLastIndexedChangeSetId(); + long lastChangeSetCommitTimeOnServer = aclTrkrState.getLastChangeSetCommitTimeOnServer(); + long lastChangeSetIdOnServer = aclTrkrState.getLastChangeSetIdOnServer(); + Date lastIndexChangeSetCommitDate = new Date(lastIndexChangeSetCommitTime); + Date lastChangeSetOnServerDate = new Date(lastChangeSetCommitTimeOnServer); + long changeSetsToDo = lastChangeSetIdOnServer - lastIndexedChangeSetId; + if (changeSetsToDo < 0) + { + changeSetsToDo = 0; + } + + long nodesToDo = 0; + long remainingTxTimeMillis = 0; + if (transactionsToDo > 0) + { + // We now use the elapsed time as seen by the single thread farming out metadata indexing + double meanDocsPerTx = srv.getTrackerStats().getMeanDocsPerTx(); + double meanNodeElaspedIndexTime = srv.getTrackerStats().getMeanNodeElapsedIndexTime(); + nodesToDo = (long)(transactionsToDo * meanDocsPerTx); + remainingTxTimeMillis = (long) (nodesToDo * meanNodeElaspedIndexTime); + } + Date now = new Date(); + Date end = new Date(now.getTime() + remainingTxTimeMillis); + Duration remainingTx = new Duration(now, end); + + long remainingChangeSetTimeMillis = 0; + if (changeSetsToDo > 0) + { + // We now use the elapsed time as seen by the single thread farming out alc indexing + double meanAclsPerChangeSet = srv.getTrackerStats().getMeanAclsPerChangeSet(); + double meanAclElapsedIndexTime = srv.getTrackerStats().getMeanAclElapsedIndexTime(); + remainingChangeSetTimeMillis = (long) (changeSetsToDo * meanAclsPerChangeSet * meanAclElapsedIndexTime); + } + now = new Date(); + end = new Date(now.getTime() + remainingChangeSetTimeMillis); + Duration remainingChangeSet = new Duration(now, end); + + NamedList ftsSummary = new SimpleOrderedMap<>(); + long remainingContentTimeMillis = 0; + srv.addFTSStatusCounts(ftsSummary); + long cleanCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Clean")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long dirtyCount = + ofNullable(ftsSummary.get("Node count with FTSStatus Dirty")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + long newCount = + ofNullable(ftsSummary.get("Node count with FTSStatus New")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long nodesInIndex = + ofNullable(coreSummary.get("Alfresco Nodes in Index")) + .map(Number.class::cast) + .map(Number::longValue) + .orElse(0L); + + long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0; + if (dirtyCount + newCount + contentYetToSee > 0) + { + // We now use the elapsed time as seen by the single thread farming out alc indexing + double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime(); + remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime); + } + now = new Date(); + end = new Date(now.getTime() + remainingContentTimeMillis); + Duration remainingContent = new Duration(now, end); + coreSummary.add("FTS",ftsSummary); + + Duration txLag = new Duration(lastIndexTxCommitDate, lastTxOnServerDate); + if (lastIndexTxCommitDate.compareTo(lastTxOnServerDate) > 0) + { + txLag = new Duration(); + } + long txLagSeconds = (lastTxCommitTimeOnServer - lastIndexTxCommitTime) / 1000; + if (txLagSeconds < 0) + { + txLagSeconds = 0; + } + + Duration changeSetLag = new Duration(lastIndexChangeSetCommitDate, lastChangeSetOnServerDate); + if (lastIndexChangeSetCommitDate.compareTo(lastChangeSetOnServerDate) > 0) + { + changeSetLag = new Duration(); + } + long changeSetLagSeconds = (lastChangeSetCommitTimeOnServer - lastIndexChangeSetCommitTime) / 1000; + if (txLagSeconds < 0) + { + txLagSeconds = 0; + } + + ContentTracker contentTrkr = trackerRegistry.getTrackerForCore(cname, ContentTracker.class); + TrackerState contentTrkrState = contentTrkr.getTrackerState(); + // Leave ModelTracker out of this check, because it is common + boolean aTrackerIsRunning = aclTrkrState.isRunning() || metadataTrkrState.isRunning() + || contentTrkrState.isRunning(); + coreSummary.add("Active", aTrackerIsRunning); + + ModelTracker modelTrkr = trackerRegistry.getModelTracker(); + TrackerState modelTrkrState = modelTrkr.getTrackerState(); + coreSummary.add("ModelTracker Active", modelTrkrState.isRunning()); + coreSummary.add("ContentTracker Active", contentTrkrState.isRunning()); + coreSummary.add("MetadataTracker Active", metadataTrkrState.isRunning()); + coreSummary.add("AclTracker Active", aclTrkrState.isRunning()); + + // TX + + coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime); + coreSummary.add("Last Index TX Commit Date", lastIndexTxCommitDate); + coreSummary.add("TX Lag", txLagSeconds + " s"); + coreSummary.add("TX Duration", txLag.toString()); + coreSummary.add("Timestamp for last TX on server", lastTxCommitTimeOnServer); + coreSummary.add("Date for last TX on server", lastTxOnServerDate); + coreSummary.add("Id for last TX on server", lastTxIdOnServer); + coreSummary.add("Id for last TX in index", lastIndexedTxId); + coreSummary.add("Approx transactions remaining", transactionsToDo); + coreSummary.add("Approx transaction indexing time remaining", remainingTx.largestComponentformattedString()); + + // Change set + + coreSummary.add("Last Index Change Set Commit Time", lastIndexChangeSetCommitTime); + coreSummary.add("Last Index Change Set Commit Date", lastIndexChangeSetCommitDate); + coreSummary.add("Change Set Lag", changeSetLagSeconds + " s"); + coreSummary.add("Change Set Duration", changeSetLag.toString()); + coreSummary.add("Timestamp for last Change Set on server", lastChangeSetCommitTimeOnServer); + coreSummary.add("Date for last Change Set on server", lastChangeSetOnServerDate); + coreSummary.add("Id for last Change Set on server", lastChangeSetIdOnServer); + coreSummary.add("Id for last Change Set in index", lastIndexedChangeSetId); + coreSummary.add("Approx change sets remaining", changeSetsToDo); + coreSummary.add("Approx change set indexing time remaining", + remainingChangeSet.largestComponentformattedString()); + + coreSummary.add("Approx content indexing time remaining", + remainingContent.largestComponentformattedString()); + + // Stats + + coreSummary.add("Model sync times (ms)", + srv.getTrackerStats().getModelTimes().getNamedList(detail, hist, values)); + coreSummary.add("Acl index time (ms)", + srv.getTrackerStats().getAclTimes().getNamedList(detail, hist, values)); + coreSummary.add("Node index time (ms)", + srv.getTrackerStats().getNodeTimes().getNamedList(detail, hist, values)); + coreSummary.add("Docs/Tx", srv.getTrackerStats().getTxDocs().getNamedList(detail, hist, values)); + coreSummary.add("Doc Transformation time (ms)", srv.getTrackerStats().getDocTransformationTimes() + .getNamedList(detail, hist, values)); + + // Model + + Map> modelErrors = srv.getModelErrors(); + if (modelErrors.size() > 0) + { + NamedList errorList = new SimpleOrderedMap<>(); + for (Map.Entry> modelNameToErrors : modelErrors.entrySet()) + { + errorList.add(modelNameToErrors.getKey(), modelNameToErrors.getValue()); + } + coreSummary.add("Model changes are not compatible with the existing data model and have not been applied", + errorList); + } + + report.add(cname, coreSummary); + } +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index 0b41cba15..f8945ef94 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -35,7 +35,7 @@ import org.alfresco.solr.tracker.CommitTracker; import org.alfresco.solr.tracker.ContentTracker; import org.alfresco.solr.tracker.MetadataTracker; import org.alfresco.solr.tracker.ModelTracker; -import org.alfresco.solr.tracker.SlaveNodeStatePublisher; +import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; @@ -182,7 +182,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener { LOGGER.info("SearchServices Core Trackers have been explicitly disabled on core \"{}\" through \"enable.alfresco.tracking\" configuration property.", core.getName()); - SlaveNodeStatePublisher statePublisher = new SlaveNodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); + SlaveCoreStatePublisher statePublisher = new SlaveCoreStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); trackerRegistry.register(core.getName(), statePublisher); scheduler.schedule(statePublisher, core.getName(), coreProperties); trackers.add(statePublisher); @@ -197,7 +197,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener { LOGGER.info("SearchServices Core Trackers have been disabled on core \"{}\" because it is a slave core.", core.getName()); - SlaveNodeStatePublisher statePublisher = new SlaveNodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); + SlaveCoreStatePublisher statePublisher = new SlaveCoreStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); trackerRegistry.register(core.getName(), statePublisher); scheduler.schedule(statePublisher, core.getName(), coreProperties); trackers.add(statePublisher); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java similarity index 91% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java index d6fd2d78a..6c38fb47a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java @@ -41,6 +41,7 @@ import org.alfresco.service.namespace.QName; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.InformationServer; +import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.commons.lang3.StringUtils; @@ -59,7 +60,7 @@ import java.util.Properties; * @since 1.5 * @see SEARCH-1752 */ -public abstract class NodeStatePublisher extends AbstractTracker +public abstract class CoreStatePublisher extends AbstractTracker { DocRouter docRouter; private final boolean isMaster; @@ -70,7 +71,7 @@ public abstract class NodeStatePublisher extends AbstractTracker /** The property to use for determining the shard. */ protected Optional shardProperty = Optional.empty(); - NodeStatePublisher( + CoreStatePublisher( boolean isMaster, Properties p, SOLRAPIClient client, @@ -88,12 +89,28 @@ public abstract class NodeStatePublisher extends AbstractTracker docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); } - NodeStatePublisher(Type type) + CoreStatePublisher(Type type) { super(type); this.isMaster = false; } + /** + * Returns information about the {@link org.alfresco.solr.client.Node} associated with the given dbid. + * + * @param dbid the node identifier. + * @return the {@link org.alfresco.solr.client.Node} associated with the given dbid. + */ + public NodeReport checkNode(Long dbid) + { + NodeReport nodeReport = new NodeReport(); + nodeReport.setDbid(dbid); + + this.infoSrv.addCommonNodeReportInfo(nodeReport); + + return nodeReport; + } + private void firstUpdateShardProperty() { shardKey.ifPresent( shardKeyName -> { @@ -222,4 +239,14 @@ public abstract class NodeStatePublisher extends AbstractTracker { return this.docRouter; } + + /** + * Returns true if the hosting core is master or standalone. + * + * @return true if the hosting core is master or standalone. + */ + public boolean isOnMasterOrStandalone() + { + return isMaster; + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index f118c523b..3862ebd30 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory; * This tracks two things: transactions and metadata nodes * @author Ahmed Owian */ -public class MetadataTracker extends NodeStatePublisher implements Tracker +public class MetadataTracker extends CoreStatePublisher implements Tracker { protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class); private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100; @@ -888,24 +888,18 @@ public class MetadataTracker extends NodeStatePublisher implements Tracker } } - - - - + @Override public NodeReport checkNode(Long dbid) { - NodeReport nodeReport = new NodeReport(); - nodeReport.setDbid(dbid); + NodeReport nodeReport = super.checkNode(dbid); // In DB - GetNodesParameters parameters = new GetNodesParameters(); parameters.setFromNodeId(dbid); parameters.setToNodeId(dbid); - List dbnodes; try { - dbnodes = client.getNodes(parameters, 1); + List dbnodes = client.getNodes(parameters, 1); if (dbnodes.size() == 1) { Node dbnode = dbnodes.get(0); @@ -915,41 +909,31 @@ public class MetadataTracker extends NodeStatePublisher implements Tracker else { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-1l); + nodeReport.setDbTx(-1L); } } catch (IOException e) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-2l); + nodeReport.setDbTx(-2L); } catch (JSONException e) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-3l); + nodeReport.setDbTx(-3L); } catch (AuthenticationException e1) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); - nodeReport.setDbTx(-4l); + nodeReport.setDbTx(-4L); } - - this.infoSrv.addCommonNodeReportInfo(nodeReport); return nodeReport; } public NodeReport checkNode(Node node) { - NodeReport nodeReport = new NodeReport(); - nodeReport.setDbid(node.getId()); - - nodeReport.setDbNodeStatus(node.getStatus()); - nodeReport.setDbTx(node.getTxnId()); - - this.infoSrv.addCommonNodeReportInfo(nodeReport); - - return nodeReport; + return checkNode(node.getId()); } public List getFullNodesForDbTransaction(Long txid) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java similarity index 90% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 4e7ccf709..19a93631c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveNodeStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -14,8 +14,8 @@ import java.util.Properties; /** * Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs - * Alfresco about the state of the hosting slave node. - * As the name suggests, this worker is scheduled only when the hosting node acts as a slave. + * Alfresco about the state of the hosting slave core. + * As the name suggests, this worker is scheduled only when the owning core acts as a slave. * It allows Solr's master/slave setup to be used with dynamic shard registration. * * In this scenario the slave is polling a "tracking" Solr node. The tracker below calls @@ -27,9 +27,9 @@ import java.util.Properties; * @author Andrea Gazzarini * @since 1.5 */ -public class SlaveNodeStatePublisher extends NodeStatePublisher +public class SlaveCoreStatePublisher extends CoreStatePublisher { - public SlaveNodeStatePublisher( + public SlaveCoreStatePublisher( boolean isMaster, Properties coreProperties, SOLRAPIClient repositoryClient, @@ -61,6 +61,12 @@ public class SlaveNodeStatePublisher extends NodeStatePublisher // Do nothing here } + @Override + public boolean isOnMasterOrStandalone() + { + return false; + } + @Override public boolean hasMaintenance() { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java new file mode 100644 index 000000000..2254b9cf0 --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.utils; + +import java.util.Collection; +import java.util.Collections; + +public abstract class Utils +{ + /** + * Returns the same input collection if that is not null, otherwise a new empty collection. + * Provides a safe way for iterating over a returned collection (which could be null). + * + * @param values the collection. + * @param the collection type. + * @return the same input collection if that is not null, otherwise a new empty collection. + */ + public static Collection notNullOrEmpty(Collection values) + { + return values != null ? values : Collections.emptyList(); + } + + /** + * Converts the given input in an Integer, otherwise it returns null. + * + * @param value the numeric string. + * @return the corresponding Integer or null in case the input is NaN. + */ + public static Integer toIntOrNull(String value) + { + try + { + return Integer.valueOf(value); + } + catch(NumberFormatException nfe) + { + return null; + } + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java index 5e264adb3..a9275c2a6 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java @@ -20,15 +20,23 @@ package org.alfresco.solr; import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ALFRESCO_CORE_NAME; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARCHIVE_CORE_NAME; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_TXID; import static org.alfresco.solr.AlfrescoCoreAdminHandler.STORE_REF_MAP; import static org.alfresco.solr.AlfrescoCoreAdminHandler.VERSION_CORE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; @@ -42,13 +50,18 @@ import java.util.stream.Collectors; import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.PropertyRouter; +import org.alfresco.solr.tracker.SlaveCoreStatePublisher; import org.alfresco.solr.tracker.TrackerRegistry; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.junit.Before; @@ -105,6 +118,169 @@ public class AlfrescoCoreAdminHandlerIT when(req.getParams()).thenReturn(params); } + @Test + public void extractShardsWithEmptyParameter_shouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.extractShards("", Integer.MAX_VALUE).isEmpty()); + } + + @Test + public void extractShardsWithNullParameter_shouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.extractShards(null, Integer.MAX_VALUE).isEmpty()); + } + + @Test + public void extractShardsWithOneInvalidShard_shouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.extractShards("This is an invalid shard id", Integer.MAX_VALUE).isEmpty()); + } + + @Test + public void extractShardsWithOneShards_shouldReturnSingletonList() + { + assertEquals(singletonList(1), alfrescoCoreAdminHandler.extractShards("1", Integer.MAX_VALUE)); + } + + @Test + public void extractShardsWithSeveralValidShards_shouldReturnAllOfThemInTheList() + { + assertEquals(asList(1,5,6,11,23), alfrescoCoreAdminHandler.extractShards("1,5,6,11,23", Integer.MAX_VALUE)); + } + + @Test + public void extractShardsWithSeveralValidShards_shouldReturnOnlyValidIdentifiers() + { + assertEquals(asList(1,5,6,11,23), alfrescoCoreAdminHandler.extractShards("1,5,A,6,xyz,11,BB,23,o01z", Integer.MAX_VALUE)); + } + + @Test + public void extractShardsWithSeveralValidShardsAndLimit_shouldConsiderOnlyShardsLesserThanLimit() + { + assertEquals(asList(1,5,6,11,12), alfrescoCoreAdminHandler.extractShards("1,5,6,11,23,25,99,223,12", 23)); + } + + @Test + public void hasAlfrescoCoreWhenInputIsNull_shouldReturnFalse() + { + assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(null)); + } + + @Test + public void hasAlfrescoCoreWhenWeHaveNoCore_shouldReturnFalse() + { + assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList())); + } + + @Test + public void hasAlfrescoCoreWhenDoesntHaveAnyTracker_shouldReturnFalse() + { + assertFalse(alfrescoCoreAdminHandler.hasAlfrescoCore(emptyList())); + } + + @Test + public void hasAlfrescoCoreWithRegisteredTrackers_shouldReturnTrue() + { + when(trackerRegistry.hasTrackersForCore("CoreD")).thenReturn(true); + assertTrue(alfrescoCoreAdminHandler.hasAlfrescoCore(asList(dummyCore("CoreA"), dummyCore("CoreB"), dummyCore("CoreC"), dummyCore("CoreD")))); + } + + @Test + public void trackerRegistryHasNoCoreNames_itShouldReturnAnEmptyList() + { + assertTrue(alfrescoCoreAdminHandler.coreNames().isEmpty()); + } + + @Test + public void coreDetectedAsMasterOrStandalone() + { + MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))) + .thenReturn(coreStatePublisher); + + assertTrue(alfrescoCoreAdminHandler.isMasterOrStandalone("ThisIsTheCoreName")); + } + + @Test + public void coreDetectedAsSlave() + { + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null); + assertFalse(alfrescoCoreAdminHandler.isMasterOrStandalone("ThisIsTheCoreName")); + } + + @Test + public void coreIsMaster_thenCoreStatePublisherInstanceCorrespondsToMetadataTracker() + { + MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))) + .thenReturn(coreStatePublisher); + + assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); + } + + @Test + public void coreIsSlave_thenCoreStatePublisherInstanceCorrespondsToSlaveCoreStatePublisher() + { + SlaveCoreStatePublisher coreStatePublisher = mock(SlaveCoreStatePublisher.class); + + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null); + when(trackerRegistry.getTrackerForCore(anyString(), eq(SlaveCoreStatePublisher.class))).thenReturn(coreStatePublisher); + + assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); + } + + @Test + public void coreIsSlave_thenDocRouterIsNull() + { + String coreName = "aCore"; + when(trackerRegistry.getTrackerForCore(eq(coreName), eq(MetadataTracker.class))).thenReturn(null); + assertNull(alfrescoCoreAdminHandler.getDocRouter("aCore")); + } + + @Test + public void coreIsMaster_thenDocRouterIsProperlyReturned() + { + DocRouter expectedRouter = new PropertyRouter("someProperty_.{1,35}"); + + MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + when(coreStatePublisher.getDocRouter()).thenReturn(expectedRouter); + when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(coreStatePublisher); + + assertSame(expectedRouter, alfrescoCoreAdminHandler.getDocRouter("aCore")); + } + + @Test + public void targetCoreNameCanBeSpecifiedInSeveralWays() + { + String coreName = "ThisIsTheCoreName"; + + ModifiableSolrParams params = new ModifiableSolrParams(); + + assertNull(alfrescoCoreAdminHandler.coreName(params)); + + params.set(CoreAdminParams.CORE, coreName); + + assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params)); + + params.remove(CoreAdminParams.CORE); + assertNull(alfrescoCoreAdminHandler.coreName(params)); + + params.set("coreName", coreName); + + assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params)); + assertEquals(coreName, alfrescoCoreAdminHandler.coreName(params)); + } + + + private SolrCore dummyCore(String name) + { + SolrCore core = mock(SolrCore.class); + when(core.getName()).thenReturn(name); + return core; + } + /** Check that a transaction report can be generated. */ @Test public void handleCustomActionTXReportSuccess() throws Exception @@ -143,8 +319,6 @@ public class AlfrescoCoreAdminHandlerIT public void handleCustomActionTXReportMissingTXId() { when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT); - when(params.get(ARG_TXID)).thenReturn(null); - alfrescoCoreAdminHandler.handleCustomAction(req, rsp); verify(rsp, never()).add(anyString(), any()); @@ -156,11 +330,8 @@ public class AlfrescoCoreAdminHandlerIT { when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT); when(params.get(CoreAdminParams.CORE)).thenReturn(null); - when(params.get(ARG_TXID)).thenReturn(TX_ID); alfrescoCoreAdminHandler.handleCustomAction(req, rsp); - - verify(rsp, never()).add(anyString(), any()); } /** Check that when an unknown action is provided we don't generate a report. */ @@ -212,10 +383,9 @@ public class AlfrescoCoreAdminHandlerIT public void coreNamesAreTrimmed_oneCoreNameAtTime() { AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() { @Override - protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) + protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) { // Do nothing here otherwise we cannot spy it - return true; } }); @@ -238,10 +408,9 @@ public class AlfrescoCoreAdminHandlerIT public void validAndInvalidCoreNames() { AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() { @Override - protected boolean newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) + protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp) { // Do nothing here otherwise we cannot spy it - return true; } });