diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..7ea71a125 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,42 @@ +# see https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "daily" + time: "22:00" + timezone: "Europe/London" + ignore: + # Solr dependencies + - dependency-name: "org.apache.lucene:lucene-analyzers-common" + - dependency-name: "org.apache.solr:solr-core" + - dependency-name: "org.apache.solr:solr-analysis-extras" + - dependency-name: "org.apache.solr:solr-langid" + - dependency-name: "org.apache.solr:solr-clustering" + - dependency-name: "org.apache.solr:solr-test-framework" + - dependency-name: "org.apache.solr:solr-solrj" + + - package-ecosystem: "docker" + directory: "search-services/packaging/src/docker/" + schedule: + interval: "weekly" + day: "saturday" + time: "22:00" + timezone: "Europe/London" + + - package-ecosystem: "docker" + directory: "insight-engine/packaging/src/docker/" + schedule: + interval: "weekly" + day: "saturday" + time: "22:00" + timezone: "Europe/London" + + - package-ecosystem: "docker" + directory: "insight-engine/alfresco-insight-zeppelin/src/docker/" + schedule: + interval: "weekly" + day: "saturday" + time: "22:00" + timezone: "Europe/London" \ No newline at end of file diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java index 2e59a6bc5..5582bf01a 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java @@ -28,6 +28,7 @@ package org.alfresco.test.search.functional.searchServices.solr.admin; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -38,6 +39,8 @@ import org.springframework.context.annotation.Configuration; import org.testng.Assert; import org.testng.annotations.Test; +import static java.util.Collections.emptyList; + /** * End to end tests for SOLR Admin actions REST API, available from: * @@ -556,9 +559,11 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("purge"); checkResponseStatusOk(response); - - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); + + DEFAULT_CORE_NAMES.forEach(core -> { + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); + Assert.assertEquals(actionStatus, "scheduled"); + }); } /** @@ -566,7 +571,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest * @throws Exception */ @Test(priority = 25) - public void testPurgeCore() throws Exception + public void testPurgeCore() { final Integer txid = 1; @@ -578,7 +583,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); Assert.assertEquals(actionStatus, "scheduled"); } catch (Exception e) @@ -599,9 +604,11 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest RestResponse response = restClient.withSolrAdminAPI().getAction("purge"); checkResponseStatusOk(response); - - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); + + DEFAULT_CORE_NAMES.forEach(core -> { + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); + Assert.assertEquals(actionStatus, "scheduled"); + }); } /** @@ -661,9 +668,11 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("reindex"); checkResponseStatusOk(response); - - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); + + DEFAULT_CORE_NAMES.forEach(core -> { + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); + Assert.assertEquals(actionStatus, "scheduled"); + }); } /** @@ -671,7 +680,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest * @throws Exception */ @Test(priority = 30) - public void testReindexCore() throws Exception + public void testReindexCore() { Integer txid = 1; @@ -683,7 +692,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); Assert.assertEquals(actionStatus, "scheduled"); } catch (Exception e) @@ -705,12 +714,12 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); - DEFAULT_CORE_NAMES.forEach(core -> { - List errorNodeList = response.getResponse().body().jsonPath().get("action." + core); - Assert.assertEquals(errorNodeList, Arrays.asList(), "Expected no error nodes,"); + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); + Assert.assertEquals(actionStatus, "scheduled"); + + List errorNodeList = response.getResponse().body().jsonPath().get("action." + core + "['Error Nodes']"); + Assert.assertEquals(errorNodeList, emptyList(), "Expected no error nodes,"); }); } @@ -719,7 +728,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest * @throws Exception */ @Test(priority = 32) - public void testRetryCore() throws Exception + public void testRetryCore() { DEFAULT_CORE_NAMES.forEach(core -> { @@ -729,11 +738,11 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); Assert.assertEquals(actionStatus, "scheduled"); - List errorNodeList = response.getResponse().body().jsonPath().get("action." + core); - Assert.assertEquals(errorNodeList, Arrays.asList(), "Expected no error nodes,"); + List errorNodeList = response.getResponse().body().jsonPath().get("action." + core + "['Error Nodes']"); + Assert.assertEquals(errorNodeList, emptyList(), "Expected no error nodes,"); } catch (Exception e) { @@ -755,9 +764,10 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("index"); checkResponseStatusOk(response); - - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); + DEFAULT_CORE_NAMES.forEach(core -> { + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); + Assert.assertEquals(actionStatus, "scheduled"); + }); } /** @@ -777,7 +787,7 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); + String actionStatus = response.getResponse().body().jsonPath().get("action." + core + ".status"); Assert.assertEquals(actionStatus, "scheduled"); } catch (Exception e) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index 19f0df1f0..ffad446ac 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -33,12 +33,12 @@ import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.SOLRAPIClientFactory; import org.alfresco.solr.config.ConfigUtil; import org.alfresco.solr.tracker.AclTracker; -import org.alfresco.solr.tracker.AbstractShardInformationPublisher; +import org.alfresco.solr.tracker.ActivatableTracker; +import org.alfresco.solr.tracker.ShardStatePublisher; import org.alfresco.solr.tracker.DBIDRangeRouter; import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; -import org.alfresco.solr.tracker.NodeStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; @@ -81,6 +81,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.LongToIntFunction; import java.util.stream.Collectors; @@ -103,6 +104,7 @@ import static org.alfresco.solr.HandlerReportHelper.buildAclTxReport; import static org.alfresco.solr.HandlerReportHelper.buildNodeReport; import static org.alfresco.solr.HandlerReportHelper.buildTrackerReport; import static org.alfresco.solr.HandlerReportHelper.buildTxReport; +import static org.alfresco.solr.utils.Utils.isNotNullAndNotEmpty; import static org.alfresco.solr.utils.Utils.isNullOrEmpty; import static org.alfresco.solr.utils.Utils.notNullOrEmpty; @@ -138,10 +140,10 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private static final String REPORT = "report"; private static final String SUMMARY = "Summary"; - private static final String ARG_ACLTXID = "acltxid"; + static final String ARG_ACLTXID = "acltxid"; static final String ARG_TXID = "txid"; - private static final String ARG_ACLID = "aclid"; - private static final String ARG_NODEID = "nodeid"; + static final String ARG_ACLID = "aclid"; + static final String ARG_NODEID = "nodeid"; private static final String ARG_QUERY = "query"; private static final String DATA_DIR_ROOT = "data.dir.root"; public static final String ALFRESCO_DEFAULTS = "create.alfresco.defaults"; @@ -167,7 +169,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private static final String ACTION_STATUS_ERROR = "error"; static final String ACTION_STATUS_SCHEDULED = "scheduled"; static final String ACTION_STATUS_NOT_SCHEDULED = "notScheduled"; - + static final String ADDITIONAL_INFO = "additionalInfo"; + static final String WARNING = "WARNING"; static final String DRY_RUN_PARAMETER_NAME = "dryRun"; static final String FROM_TX_COMMIT_TIME_PARAMETER_NAME = "fromTxCommitTime"; static final String TO_TX_COMMIT_TIME_PARAMETER_NAME = "toTxCommitTime"; @@ -197,7 +200,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler TrackerRegistry trackerRegistry; ConcurrentHashMap informationServers; - private final static List CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index"); + final static List CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index"); public AlfrescoCoreAdminHandler() { @@ -495,6 +498,14 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler ofNullable(params.get("resource")) .orElse("log4j.properties"))); break; + case "ENABLE-INDEXING": + case "ENABLEINDEXING": + rsp.add(ACTION_LABEL, actionEnableIndexing(params)); + break; + case "DISABLE-INDEXING": + case "DISABLEINDEXING": + rsp.add(ACTION_LABEL, actionDisableIndexing(params)); + break; default: super.handleCustomAction(req, rsp); break; @@ -1388,8 +1399,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * - toCalTx, optional: to ACL transaction Id to filter report results * * - report.core: multiple Objects with the details of the report ("core" is the name of the Core) - * - * @throws JSONException */ private NamedList actionREPORT(SolrParams params) throws JSONException { @@ -1444,16 +1453,30 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @return Response including the action result: * - status: scheduled, as it will be executed by Trackers on the next maintenance operation */ - private NamedList actionPURGE(SolrParams params) + NamedList actionPURGE(SolrParams params) { + final NamedList response = new SimpleOrderedMap<>(); Consumer purgeOnSpecificCore = coreName -> { final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - apply(params, ARG_TXID, metadataTracker::addTransactionToPurge); - apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToPurge); - apply(params, ARG_NODEID, metadataTracker::addNodeToPurge); - apply(params, ARG_ACLID, aclTracker::addAclToPurge); + final NamedList coreResponse = new SimpleOrderedMap<>(); + + if (metadataTracker.isEnabled() & aclTracker.isEnabled()) + { + apply(params, ARG_TXID, metadataTracker::addTransactionToPurge); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToPurge); + apply(params, ARG_NODEID, metadataTracker::addNodeToPurge); + apply(params, ARG_ACLID, aclTracker::addAclToPurge); + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + } + else + { + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_NOT_SCHEDULED); + coreResponse.add(ADDITIONAL_INFO, "Trackers have been disabled: the purge request cannot be executed; please enable indexing and then resubmit this command."); + } + + response.add(coreName, coreResponse); }; String requestedCoreName = coreName(params); @@ -1463,8 +1486,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .filter(this::isMasterOrStandalone) .forEach(purgeOnSpecificCore); - NamedList response = new SimpleOrderedMap<>(); - response.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + if (response.size() == 0) + { + addAlertMessage(response); + } + return response; } @@ -1484,18 +1510,31 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @return Response including the action result: * - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation */ - private NamedList actionREINDEX(SolrParams params) + NamedList actionREINDEX(SolrParams params) { + final NamedList response = new SimpleOrderedMap<>(); Consumer reindexOnSpecificCore = coreName -> { final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + final NamedList coreResponse = new SimpleOrderedMap<>(); - apply(params, ARG_TXID, metadataTracker::addTransactionToReindex); - apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToReindex); - apply(params, ARG_NODEID, metadataTracker::addNodeToReindex); - apply(params, ARG_ACLID, aclTracker::addAclToReindex); + if (metadataTracker.isEnabled() & aclTracker.isEnabled()) + { + apply(params, ARG_TXID, metadataTracker::addTransactionToReindex); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToReindex); + apply(params, ARG_NODEID, metadataTracker::addNodeToReindex); + apply(params, ARG_ACLID, aclTracker::addAclToReindex); - ofNullable(params.get(ARG_QUERY)).ifPresent(metadataTracker::addQueryToReindex); + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + ofNullable(params.get(ARG_QUERY)).ifPresent(metadataTracker::addQueryToReindex); + } + else + { + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_NOT_SCHEDULED); + coreResponse.add(ADDITIONAL_INFO, "Trackers have been disabled: the REINDEX request cannot be executed; please enable indexing and then resubmit this command."); + } + + response.add(coreName, coreResponse); }; String requestedCoreName = coreName(params); @@ -1505,8 +1544,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .filter(this::isMasterOrStandalone) .forEach(reindexOnSpecificCore); - NamedList response = new SimpleOrderedMap<>(); - response.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + if (response.size() == 0) + { + addAlertMessage(response); + } + return response; } @@ -1520,35 +1562,41 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation * - core: list of Document Ids with error that are going to reindexed */ - private NamedList actionRETRY(SolrParams params) + NamedList actionRETRY(SolrParams params) { NamedList response = new SimpleOrderedMap<>(); final Consumer retryOnSpecificCore = coreName -> { MetadataTracker tracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); InformationServer srv = informationServers.get(coreName); + final NamedList coreResponse = new SimpleOrderedMap<>(); - try + if (tracker.isEnabled()) { - for (Long nodeid : srv.getErrorDocIds()) + try { - tracker.addNodeToReindex(nodeid); + for (Long nodeid : srv.getErrorDocIds()) + { + tracker.addNodeToReindex(nodeid); + } + coreResponse.add("Error Nodes", srv.getErrorDocIds()); + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + } catch (Exception exception) + { + LOGGER.error("I/O Exception while adding Node to reindex.", exception); + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_ERROR); + coreResponse.add(ACTION_ERROR_MESSAGE_LABEL, exception.getMessage()); + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_NOT_SCHEDULED); } - response.add(coreName, srv.getErrorDocIds()); } - catch (Exception exception) + else { - LOGGER.error("I/O Exception while adding Node to reindex.", exception); - response.add(ACTION_STATUS_LABEL, ACTION_STATUS_ERROR); - response.add(ACTION_ERROR_MESSAGE_LABEL, exception.getMessage()); - + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_NOT_SCHEDULED); + coreResponse.add(ADDITIONAL_INFO, "Trackers have been disabled: the RETRY request cannot be executed; please enable indexing and then resubmit this command."); } - }; - if (Objects.equals(response.get(ACTION_STATUS_LABEL), ACTION_STATUS_ERROR)) - { - return response; - } + response.add(coreName, coreResponse); + }; String requestedCoreName = coreName(params); @@ -1557,7 +1605,11 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .filter(this::isMasterOrStandalone) .forEach(retryOnSpecificCore); - response.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + if (response.size() == 0) + { + addAlertMessage(response); + } + return response; } @@ -1576,16 +1628,29 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @return Response including the action result: * - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation */ - private NamedList actionINDEX(SolrParams params) + NamedList actionINDEX(SolrParams params) { + final NamedList response = new SimpleOrderedMap<>(); Consumer indexOnSpecificCore = coreName -> { final MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); final AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + final NamedList coreResponse = new SimpleOrderedMap<>(); - apply(params, ARG_TXID, metadataTracker::addTransactionToIndex); - apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToIndex); - apply(params, ARG_NODEID, metadataTracker::addNodeToIndex); - apply(params, ARG_ACLID, aclTracker::addAclToIndex); + if (metadataTracker.isEnabled() & aclTracker.isEnabled()) + { + apply(params, ARG_TXID, metadataTracker::addTransactionToIndex); + apply(params, ARG_ACLTXID, aclTracker::addAclChangeSetToIndex); + apply(params, ARG_NODEID, metadataTracker::addNodeToIndex); + apply(params, ARG_ACLID, aclTracker::addAclToIndex); + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + } + else + { + coreResponse.add(ACTION_STATUS_LABEL, ACTION_STATUS_NOT_SCHEDULED); + coreResponse.add(ADDITIONAL_INFO, "Trackers have been disabled: the INDEX request cannot be executed; please enable indexing and then resubmit this command."); + } + + response.add(coreName, coreResponse); }; String requestedCoreName = coreName(params); @@ -1595,11 +1660,24 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .filter(this::isMasterOrStandalone) .forEach(indexOnSpecificCore); - NamedList response = new SimpleOrderedMap<>(); - response.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); + if (response.size() == 0) + { + addAlertMessage(response); + } + return response; } + NamedList actionDisableIndexing(SolrParams params) throws JSONException + { + return executeTrackerSubsystemLifecycleAction(params, this::disableIndexingOnSpecificCore); + } + + NamedList actionEnableIndexing(SolrParams params) throws JSONException + { + return executeTrackerSubsystemLifecycleAction(params, this::enableIndexingOnSpecificCore); + } + /** * Find transactions and acls missing or duplicated in the cores and * add them to be reindexed on the next maintenance operation @@ -1644,24 +1722,30 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler boolean dryRun = params.getBool(DRY_RUN_PARAMETER_NAME, true); int maxTransactionsToSchedule = getMaxTransactionToSchedule(params); + MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(requestedCoreName, MetadataTracker.class); + AclTracker aclTracker = trackerRegistry.getTrackerForCore(requestedCoreName, AclTracker.class); + final boolean actualDryRun = dryRun | (metadataTracker == null || metadataTracker.isDisabled()) || (aclTracker == null || aclTracker.isDisabled()); + LOGGER.debug("FIX Admin request on core {}, parameters: " + FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + DRY_RUN_PARAMETER_NAME + " = {}, " + - MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}", + "actualDryRun = {} " + + MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}", requestedCoreName, ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."), ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."), dryRun, + actualDryRun, maxTransactionsToSchedule); coreNames().stream() - .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(coreName -> coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) .forEach(coreName -> wrapper.response.add( coreName, - fixOnSpecificCore(coreName, fromTxCommitTime, toTxCommitTime, dryRun, maxTransactionsToSchedule))); + fixOnSpecificCore(coreName, fromTxCommitTime, toTxCommitTime, actualDryRun, maxTransactionsToSchedule))); if (wrapper.response.size() > 0) { @@ -1671,7 +1755,14 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler ofNullable(toTxCommitTime).ifPresent(value -> wrapper.response.add(TO_TX_COMMIT_TIME_PARAMETER_NAME, value)); wrapper.response.add(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME, maxTransactionsToSchedule); - wrapper.response.add(ACTION_STATUS_LABEL, dryRun ? ACTION_STATUS_NOT_SCHEDULED : ACTION_STATUS_SCHEDULED); + wrapper.response.add(ACTION_STATUS_LABEL, actualDryRun ? ACTION_STATUS_NOT_SCHEDULED : ACTION_STATUS_SCHEDULED); + + // the user wanted a real execution (dryRun = false) but the trackers are disabled. + // that adds a message in the response just to inform the user we didn't schedule anything (i.e. we forced a dryRun) + if (!dryRun && actualDryRun) + { + wrapper.response.add(ADDITIONAL_INFO, "Trackers are disabled: a (dryRun = true) has been forced. As consequence of that nothing has been scheduled."); + } } return wrapper.response; @@ -1701,6 +1792,8 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler try { MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); + AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); + final IndexHealthReport metadataTrackerIndexHealthReport = metadataTracker.checkIndex(null, fromTxCommitTime, toTxCommitTime); @@ -1715,7 +1808,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler dryRun, maxTransactionsToSchedule); - AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); final IndexHealthReport aclTrackerIndexHealthReport = aclTracker.checkIndex(null, fromTxCommitTime, toTxCommitTime); @@ -2009,11 +2101,9 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * @param coreName the owning core name. * @return the component which is in charge to publish the core state. */ - AbstractShardInformationPublisher coreStatePublisher(String coreName) + ShardStatePublisher coreStatePublisher(String coreName) { - return ofNullable(trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class)) - .map(AbstractShardInformationPublisher.class::cast) - .orElse(trackerRegistry.getTrackerForCore(coreName, NodeStatePublisher.class)); + return trackerRegistry.getTrackerForCore(coreName, ShardStatePublisher.class); } /** @@ -2036,7 +2126,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler private void addAlertMessage(NamedList report) { report.add( - "WARNING", + WARNING, "The requested endpoint is not available on the slave. " + "Please re-submit the same request to the corresponding Master"); } @@ -2084,4 +2174,62 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .map(Integer::parseInt) .orElse(Integer.MAX_VALUE)); // Last fallback if we don't have a request param and a value in configuration } + + NamedList disableIndexingOnSpecificCore(String coreName) { + final NamedList coreResponse = new SimpleOrderedMap<>(); + trackerRegistry.getTrackersForCore(coreName) + .stream() + .filter(tracker -> tracker instanceof ActivatableTracker) + .map(ActivatableTracker.class::cast) + .peek(ActivatableTracker::disable) + .forEach(tracker -> coreResponse.add(tracker.getType().toString(), tracker.isEnabled())); + return coreResponse; + } + + NamedList enableIndexingOnSpecificCore(String coreName) { + final NamedList coreResponse = new SimpleOrderedMap<>(); + trackerRegistry.getTrackersForCore(coreName) + .stream() + .filter(tracker -> tracker instanceof ActivatableTracker) + .map(ActivatableTracker.class::cast) + .peek(ActivatableTracker::enable) + .forEach(tracker -> coreResponse.add(tracker.getType().toString(), tracker.isEnabled())); + return coreResponse; + } + + /** + * Internal method used for executing the enable/disable indexing/tracking action. + * + * @param params the input request parameters. The only mandatory parameter is the core name + * @param action this can be the "enable" or the "disable" action: it is an "impure" function which takes a core name + * executes the enable/disable logic as part of its side-effect, and returns the action response. + * @return the action response indicating the result of the enable/disable command on a specific core. + * @see #CORE_PARAMETER_NAMES + */ + private NamedList executeTrackerSubsystemLifecycleAction(SolrParams params, Function> action) throws JSONException + { + String requestedCoreName = coreName(params); + final NamedList response = new SimpleOrderedMap<>(); + + if (isNotNullAndNotEmpty(requestedCoreName)) + { + if (!coreNames().contains(requestedCoreName)) + { + response.add(ACTION_ERROR_MESSAGE_LABEL, UNKNOWN_CORE_MESSAGE + requestedCoreName); + return response; + } + + if (!isMasterOrStandalone(requestedCoreName)) { + response.add(ACTION_ERROR_MESSAGE_LABEL, UNPROCESSABLE_REQUEST_ON_SLAVE_NODES); + return response; + } + } + + coreNames().stream() + .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) + .filter(this::isMasterOrStandalone) + .forEach(coreName -> response.add(coreName, action.apply(coreName))); + + return response; + } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java index ebcf4ef26..e30f8b8ce 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java @@ -125,7 +125,7 @@ class HandlerReportHelper return nr; } - static NamedList buildNodeReport(AbstractShardInformationPublisher publisher, Long dbid) throws JSONException + static NamedList buildNodeReport(ShardStatePublisher publisher, Long dbid) throws JSONException { NodeReport nodeReport = publisher.checkNode(dbid); @@ -159,6 +159,7 @@ class HandlerReportHelper AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); IndexHealthReport aclReport = aclTracker.checkIndex(toAclTx, fromTime, toTime); NamedList ihr = new SimpleOrderedMap<>(); + ihr.add("ACL Tracker", (aclTracker.isEnabled() ? "enabled" : "disabled")); ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() .cardinality()); @@ -188,6 +189,7 @@ class HandlerReportHelper // Metadata MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, fromTime, toTime); + ihr.add("Metadata Tracker", (metadataTracker.isEnabled() ? "enabled" : "disabled")); ihr.add("DB transaction count", metaReport.getDbTransactionCount()); ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex() .cardinality()); @@ -248,7 +250,7 @@ class HandlerReportHelper NamedList coreSummary = new SimpleOrderedMap<>(); coreSummary.addAll((SimpleOrderedMap) srv.getCoreStats()); - NodeStatePublisher statePublisher = trackerRegistry.getTrackerForCore(cname, NodeStatePublisher.class); + ShardStatePublisher statePublisher = trackerRegistry.getTrackerForCore(cname, ShardStatePublisher.class); TrackerState trackerState = statePublisher.getTrackerState(); long lastIndexTxCommitTime = trackerState.getLastIndexedTxCommitTime(); @@ -429,17 +431,12 @@ class HandlerReportHelper long remainingContentTimeMillis = 0; srv.addContentOutdatedAndUpdatedCounts(ftsSummary); long cleanCount = - ofNullable(ftsSummary.get("Node count with FTSStatus Clean")) + ofNullable(ftsSummary.get("Node count whose content is in sync")) .map(Number.class::cast) .map(Number::longValue) .orElse(0L); long dirtyCount = - ofNullable(ftsSummary.get("Node count with FTSStatus Dirty")) - .map(Number.class::cast) - .map(Number::longValue) - .orElse(0L); - long newCount = - ofNullable(ftsSummary.get("Node count with FTSStatus New")) + ofNullable(ftsSummary.get("Node count whose content needs to be updated")) .map(Number.class::cast) .map(Number::longValue) .orElse(0L); @@ -450,12 +447,14 @@ class HandlerReportHelper .map(Number::longValue) .orElse(0L); - long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount + newCount)/nodesInIndex : 0; - if (dirtyCount + newCount + contentYetToSee > 0) + + + long contentYetToSee = nodesInIndex > 0 ? nodesToDo * (cleanCount + dirtyCount)/nodesInIndex : 0; + if (dirtyCount + contentYetToSee > 0) { // We now use the elapsed time as seen by the single thread farming out alc indexing double meanContentElapsedIndexTime = srv.getTrackerStats().getMeanContentElapsedIndexTime(); - remainingContentTimeMillis = (long) ((dirtyCount + newCount + contentYetToSee) * meanContentElapsedIndexTime); + remainingContentTimeMillis = (long) ((dirtyCount + contentYetToSee) * meanContentElapsedIndexTime); } now = new Date(); end = new Date(now.getTime() + remainingContentTimeMillis); @@ -485,6 +484,8 @@ class HandlerReportHelper } ContentTracker contentTrkr = trackerRegistry.getTrackerForCore(cname, ContentTracker.class); + CascadeTracker cascadeTracker = trackerRegistry.getTrackerForCore(cname, CascadeTracker.class); + TrackerState contentTrkrState = contentTrkr.getTrackerState(); // Leave ModelTracker out of this check, because it is common boolean aTrackerIsRunning = aclTrkrState.isRunning() || metadataTrkrState.isRunning() @@ -498,6 +499,11 @@ class HandlerReportHelper coreSummary.add("MetadataTracker Active", metadataTrkrState.isRunning()); coreSummary.add("AclTracker Active", aclTrkrState.isRunning()); + coreSummary.add("ContentTracker Enabled", contentTrkr.isEnabled()); + coreSummary.add("MetadataTracker Enabled", metaTrkr.isEnabled()); + coreSummary.add("AclTracker Enabled", aclTrkr.isEnabled()); + coreSummary.add("CascadeTracker Enabled", cascadeTracker.isEnabled()); + // TX coreSummary.add("Last Index TX Commit Time", lastIndexTxCommitTime); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFieldListComponent.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFieldListComponent.java index 283070d32..c539c3c45 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFieldListComponent.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFieldListComponent.java @@ -102,7 +102,7 @@ public class RewriteFieldListComponent extends SearchComponent { { fieldListSet.add("*"); } - else + else if (solrReturnFields.getLuceneFieldNames() != null) { fieldListSet.addAll(solrReturnFields.getLuceneFieldNames().stream() .map( field -> AlfrescoSolrDataModel.getInstance() diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index 00cb220d4..afb1e8376 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -30,6 +30,7 @@ import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; import static org.alfresco.solr.SolrInformationServer.CASCADE_TRACKER_ENABLED; +import static org.alfresco.solr.tracker.ActivatableTracker.INDEXING_ENABLED_PERSISTENT_FLAG_ACROSS_RELOADS; import java.util.ArrayList; import java.util.Collection; @@ -46,12 +47,13 @@ import org.alfresco.solr.SolrKeyResourceLoader; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.client.SOLRAPIClientFactory; import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.ActivatableTracker; import org.alfresco.solr.tracker.CascadeTracker; import org.alfresco.solr.tracker.CommitTracker; import org.alfresco.solr.tracker.ContentTracker; import org.alfresco.solr.tracker.MetadataTracker; import org.alfresco.solr.tracker.ModelTracker; -import org.alfresco.solr.tracker.NodeStatePublisher; +import org.alfresco.solr.tracker.ShardStatePublisher; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; import org.alfresco.solr.tracker.TrackerRegistry; @@ -190,7 +192,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener { LOGGER.info("SearchServices Core Trackers have been explicitly disabled on core \"{}\" through \"enable.alfresco.tracking\" configuration property.", core.getName()); - NodeStatePublisher statePublisher = new NodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); + ShardStatePublisher statePublisher = new ShardStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); trackerRegistry.register(core.getName(), statePublisher); scheduler.schedule(statePublisher, core.getName(), coreProperties); trackers.add(statePublisher); @@ -205,7 +207,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener { LOGGER.info("SearchServices Core Trackers have been disabled on core \"{}\" because it is a slave core.", core.getName()); - NodeStatePublisher statePublisher = new NodeStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); + ShardStatePublisher statePublisher = new ShardStatePublisher(false, coreProperties, repositoryClient, core.getName(), informationServer); trackerRegistry.register(core.getName(), statePublisher); scheduler.schedule(statePublisher, core.getName(), coreProperties); trackers.add(statePublisher); @@ -264,9 +266,9 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener trackerRegistry, scheduler); - NodeStatePublisher coreStateTracker = + ShardStatePublisher coreStateTracker = registerAndSchedule( - new NodeStatePublisher(true, props, repositoryClient, core.getName(), srv), + new ShardStatePublisher(true, props, repositoryClient, core.getName(), srv), core, props, trackerRegistry, @@ -288,6 +290,22 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener trackers.add(cascadeTracker); } + boolean indexingHasBeenEnabled = + Boolean.parseBoolean(props.getProperty(INDEXING_ENABLED_PERSISTENT_FLAG_ACROSS_RELOADS, "true")); + + if (indexingHasBeenEnabled) + { + trackers.stream().map(ActivatableTracker.class::cast).forEach(ActivatableTracker::enable); + LOGGER.info("SearchServices Core trackers (i.e. indexing) have been enabled."); + } + else + { + trackers.stream().map(ActivatableTracker.class::cast).forEach(ActivatableTracker::disable); + LOGGER.info("SearchServices Core trackers (i.e. indexing) have been disabled. That could happen if you " + + "previously disabled the indexing on this core and then you reloaded it. If you want to enable indexing " + + "please invoke the \"enable-indexing\" admin action."); + } + //The CommitTracker will acquire these locks in order //The ContentTracker will likely have the longest runs so put it first to ensure the MetadataTracker is not paused while //waiting for the ContentTracker to release it's lock. diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index 671c14b4a..f020f71a5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -29,19 +29,29 @@ package org.alfresco.solr.tracker; import static java.util.Optional.ofNullable; import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID; +import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_KEY_KEY; import java.net.ConnectException; import java.net.SocketTimeoutException; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.Semaphore; -import java.util.function.Consumer; +import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; +import org.alfresco.repo.dictionary.NamespaceDAO; import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.repo.search.impl.QueryParserUtils; +import org.alfresco.service.cmr.dictionary.DictionaryService; +import org.alfresco.service.cmr.dictionary.PropertyDefinition; import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.namespace.QName; +import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.IndexTrackingShutdownException; import org.alfresco.solr.InformationServer; +import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,6 +91,24 @@ public abstract class AbstractTracker implements Tracker protected final Type type; protected final String trackerId; + DocRouter docRouter; + + /** + * The property to use for determining the shard. + * Note that this property is not used by all trackers, it is actually managed by the {@link ShardStatePublisher} and + * {@link MetadataTracker}. We put this property here because otherwise we should introduce another supertype layer + * for those two trackers. + */ + protected Optional shardProperty = Optional.empty(); + + /** + * The string representation of the shard key. + * Note that this property is not used by all trackers, it is actually managed by the {@link ShardStatePublisher} and + * {@link MetadataTracker}. We put this property here because otherwise we should introduce another supertype layer + * for those two trackers. + */ + protected Optional shardKey; + /** * Default constructor, strictly for testing. */ @@ -114,9 +142,14 @@ public abstract class AbstractTracker implements Tracker this.type = type; this.trackerId = type + "@" + hashCode(); + + shardKey = ofNullable(p.getProperty(SHARD_KEY_KEY)); + + firstUpdateShardProperty(); + + docRouter = DocRouterFactory.getRouter(p, shardMethod); } - /** * Subclasses must implement behaviour that completes the following steps, in order: * @@ -352,4 +385,88 @@ public abstract class AbstractTracker implements Tracker { return type; } + + /** + * Set the shard property using the shard key. + */ + void updateShardProperty() + { + shardKey.ifPresent(shardKeyName -> { + Optional updatedShardProperty = getShardProperty(shardKeyName); + if (!shardProperty.equals(updatedShardProperty)) + { + if (updatedShardProperty.isEmpty()) + { + LOGGER.warn("The model defining {} property has been disabled", shardKeyName); + } + else + { + LOGGER.info("New {} property found for {}", SHARD_KEY_KEY, shardKeyName); + } + } + shardProperty = updatedShardProperty; + }); + } + + /** + * Given the field name, returns the name of the property definition. + * If the property definition is not found, Empty optional is returned. + * + * @param field the field name. + * @return the name of the associated property definition if present, Optional.Empty() otherwise + */ + static Optional getShardProperty(String field) + { + if (StringUtils.isBlank(field)) + { + throw new IllegalArgumentException("Sharding property " + SHARD_KEY_KEY + " has not been set."); + } + + AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance(); + NamespaceDAO namespaceDAO = dataModel.getNamespaceDAO(); + DictionaryService dictionaryService = dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT); + PropertyDefinition propertyDef = QueryParserUtils.matchPropertyDefinition("http://www.alfresco.org/model/content/1.0", + namespaceDAO, + dictionaryService, + field); + + return ofNullable(propertyDef).map(PropertyDefinition::getName); + } + + /** + * Returns information about the {@link org.alfresco.solr.client.Node} associated with the given dbid. + * + * @param dbid the node identifier. + * @return the {@link org.alfresco.solr.client.Node} associated with the given dbid. + */ + public NodeReport checkNode(Long dbid) + { + NodeReport nodeReport = new NodeReport(); + nodeReport.setDbid(dbid); + + this.infoSrv.addCommonNodeReportInfo(nodeReport); + + return nodeReport; + } + + /** + * Returns the {@link DocRouter} instance in use on this node. + * + * @return the {@link DocRouter} instance in use on this node. + */ + public DocRouter getDocRouter() + { + return this.docRouter; + } + + private void firstUpdateShardProperty() + { + shardKey.ifPresent( shardKeyName -> { + updateShardProperty(); + if (shardProperty.isEmpty()) + { + LOGGER.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); + } + }); + } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index 1d0d72c62..0975904cf 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -67,7 +67,7 @@ import org.slf4j.LoggerFactory; * @author Matt Ward **/ -public class AclTracker extends AbstractTracker +public class AclTracker extends ActivatableTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class); @@ -363,6 +363,19 @@ public class AclTracker extends AbstractTracker aclsToPurge.offer(aclToPurge); } + @Override + protected void clearScheduledMaintenanceWork() + { + logAndClear(aclChangeSetsToIndex, "ACL ChangeSets to be indexed"); + logAndClear(aclsToIndex, "ACLs to be indexed"); + + logAndClear(aclChangeSetsToReindex, "ACL ChangeSets to be re-indexed"); + logAndClear(aclsToReindex, "ACLs to be re-indexed"); + + logAndClear(aclChangeSetsToPurge, "ACL ChangeSets to be purged"); + logAndClear(aclsToPurge, "ACLs to be purged"); + } + protected void trackRepository() throws IOException, AuthenticationException, JSONException { checkShutdown(); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ActivatableTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ActivatableTracker.java new file mode 100644 index 000000000..251635ccb --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ActivatableTracker.java @@ -0,0 +1,165 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.solr.tracker; + +import org.alfresco.solr.InformationServer; +import org.alfresco.solr.client.SOLRAPIClient; +import org.apache.solr.core.CoreDescriptorDecorator; +import org.apache.solr.core.SolrCore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Supertype layer for trackers that can be enabled/disabled. + */ +public abstract class ActivatableTracker extends AbstractTracker +{ + private static final Logger LOGGER = LoggerFactory.getLogger(ActivatableTracker.class); + public final static String INDEXING_ENABLED_PERSISTENT_FLAG_ACROSS_RELOADS = "alfresco.trackers.indexingEnabled"; + + protected final AtomicBoolean isEnabled; + + protected ActivatableTracker(Type type) + { + super(type); + this.isEnabled = new AtomicBoolean(true); + } + + protected ActivatableTracker(Properties properties, SOLRAPIClient client, String coreName, InformationServer informationServer, Type type) + { + super(properties, client, coreName, informationServer, type); + isEnabled = new AtomicBoolean(Boolean.parseBoolean(properties.getProperty(INDEXING_ENABLED_PERSISTENT_FLAG_ACROSS_RELOADS, "true"))); + + if (isEnabled.get()) + { + LOGGER.info("[{} / {} / {}] {} Tracker set to enabled at startup.", coreName, trackerId, state, type); + } + else + { + LOGGER.info("[{} / {} / {}] {} Tracker set to disabled at startup.", coreName, trackerId, state, type); + } + } + + protected void setPersistentIndexingStateAcrossReloadsTo(boolean enabled) + { + infoSrv.getAdminHandler().getCoreContainer().getCoreDescriptor(coreName).setProperty(INDEXING_ENABLED_PERSISTENT_FLAG_ACROSS_RELOADS, String.valueOf(enabled)); + } + + /** + * Disables this tracker instance. + */ + public final void disable() + { + clearScheduledMaintenanceWork(); + + if (isEnabled.compareAndSet(true, false)) + { + setPersistentIndexingStateAcrossReloadsTo(false); + if (state != null && state.isRunning()) + { + LOGGER.info("[{} / {} / {}] {} Tracker has been disabled (the change will be effective at the next tracking cycle) and set in rollback mode because it is running.", coreName, trackerId, state, type); + setRollback(true, null); + } + LOGGER.info("[{} / {} / {}] {} Tracker has been disabled. The change will be effective at the next tracking cycle.", coreName, trackerId, state, type); + } + else + { + LOGGER.warn("[{} / {} / {}] {} Tracker cannot be disabled because it is already in that state.", coreName, trackerId, state, type); + } + } + + /** + * Enables this tracker instance. + */ + public final void enable() + { + if (isEnabled.compareAndSet(false, true)) + { + setPersistentIndexingStateAcrossReloadsTo(true); + LOGGER.info("[{} / {} / {}] {} Tracker has been enabled. The change will be effective at the next tracking cycle.", coreName, trackerId, state, type); + } + else + { + LOGGER.warn("[{} / {} / {}] {} Tracker cannot be enabled because it is already in that state.", coreName, trackerId, state, type); + } + } + + @Override + public void track() + { + if (isEnabled()) + { + super.track(); + } + else + { + LOGGER.trace("[{} / {} / {}] {} Tracker is disabled. That is absolutely ok, that means you disabled the tracking on this core.", coreName, trackerId, state, type); + } + } + + public boolean isEnabled() + { + return isEnabled.get(); + } + + public boolean isDisabled() + { + return !isEnabled(); + } + + /** + * Cleans up the scheduled maintenance work collected by this tracker. + */ + protected void clearScheduledMaintenanceWork() + { + // Default behaviour is: do nothing + }; + + /** + * Logs out the content of the input collection. + * + * @param values the collection which (in case is not empty) contains the identifiers (e.g. txid, aclid) the system + * is going to clear. + * @param kind the kind of identifier (e.g. Transaction, Node ID, ACL ID) in the input collection. + */ + protected void logAndClear(Collection values, String kind) + { + if (values == null || values.size() == 0) { + return; + } + + final List tmp = new ArrayList<>(values); + values.clear(); + + LOGGER.info("[CORE {}] Scheduled work ({}) that will be cleaned: {}", coreName, kind, new ArrayList<>(tmp)); + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 96751218b..df0eda7af 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -52,14 +52,11 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.util.stream.Collectors.joining; -import static org.alfresco.solr.utils.Utils.notNullOrEmpty; - /* * This tracks Cascading Updates * @author Joel Bernstein */ -public class CascadeTracker extends AbstractTracker implements Tracker +public class CascadeTracker extends ActivatableTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(CascadeTracker.class); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index 7066b2907..b98995ddb 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -112,12 +112,13 @@ public class CommitTracker extends AbstractTracker WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } - public boolean hasMaintenance() throws Exception + public boolean hasMaintenance() { return (metadataTracker.hasMaintenance() || aclTracker.hasMaintenance()); } - public int getRollbackCount() { + public int getRollbackCount() + { return rollbackCount.get(); } @@ -134,7 +135,6 @@ public class CommitTracker extends AbstractTracker boolean commitNeeded = false; boolean openSearcherNeeded = false; boolean hasMaintenance = hasMaintenance(); - //System.out.println("############# Commit Tracker doTrack()"); if((currentTime - lastCommit) > commitInterval || hasMaintenance) { @@ -151,8 +151,6 @@ public class CommitTracker extends AbstractTracker openSearcherNeeded = true; } - //System.out.println("############# Commit Tracker commit needed"); - try { metadataTracker.getWriteLock().acquire(); @@ -161,9 +159,8 @@ public class CommitTracker extends AbstractTracker aclTracker.getWriteLock().acquire(); assert(aclTracker.getWriteLock().availablePermits() == 0); - //See if we need a rollback - if(metadataTracker.getRollback() || aclTracker.getRollback()) { - + if(metadataTracker.getRollback() || aclTracker.getRollback()) + { /* * The metadataTracker and aclTracker will return true if an unhandled exception has occurred during indexing. * @@ -174,30 +171,36 @@ public class CommitTracker extends AbstractTracker * the index, rather then the in-memory state. This keeps the trackers in-sync with index if their work is * rolled back. */ - doRollback(); return; } - if(hasMaintenance) { + // The disable-indexing command should happen while the commit tracker is here. + // In that case the (disable-indexing) command clears the maintenance work as much as possible, + // however, there's a chance that some work will be still executed: for that reason the check is repeated + // later (see below) and in case a rollback is executed + if (hasMaintenance) + { maintenance(); } - //Do the commit opening the searcher if needed. This will commit all the work done by indexing trackers. - //This will return immediately and not wait for searchers to warm - boolean searcherOpened = infoSrv.commit(openSearcherNeeded); - - lastCommit = currentTime; - if(searcherOpened) { - lastSearcherOpened = currentTime; + if (metadataTracker.isEnabled() && aclTracker.isEnabled()) + { + boolean searcherOpened = infoSrv.commit(openSearcherNeeded); + lastCommit = currentTime; + if(searcherOpened) + { + lastSearcherOpened = currentTime; + } + } + else + { + doRollback(); } } finally { - //Release the lock on the metadata Tracker metadataTracker.getWriteLock().release(); - - //Release the lock on the aclTracker aclTracker.getWriteLock().release(); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java index b260fa031..8f77105e7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java @@ -49,7 +49,7 @@ import static org.alfresco.solr.utils.Utils.notNullOrEmpty; * * @author Ahmed Owian */ -public class ContentTracker extends AbstractTracker implements Tracker +public class ContentTracker extends ActivatableTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(ContentTracker.class); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 383992ab5..cbccb83c0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -68,7 +68,7 @@ import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE; * This tracks two things: transactions and metadata nodes * @author Ahmed Owian */ -public class MetadataTracker extends AbstractShardInformationPublisher implements Tracker +public class MetadataTracker extends ActivatableTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(MetadataTracker.class); @@ -101,8 +101,8 @@ public class MetadataTracker extends AbstractShardInformationPublisher implement private ForkJoinPool forkJoinPool; // Share run and write locks across all MetadataTracker threads - private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static final Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static final Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); @Override public Semaphore getWriteLock() { @@ -119,7 +119,7 @@ public class MetadataTracker extends AbstractShardInformationPublisher implement * This service is used to find the next available transaction commit time from a given time, * so periods of time where no document updating is happening can be skipped while getting * pending transactions list. - * + * * {@link org.alfresco.solr.client.SOLRAPIClient#GET_NEXT_TX_COMMIT_TIME} */ private boolean nextTxCommitTimeServiceAvailable = false; @@ -127,8 +127,8 @@ public class MetadataTracker extends AbstractShardInformationPublisher implement /** * Check if txInteravlCommitTimeService is available in the repository. * This service returns the minimum and the maximum commit time for transactions in a node id range, - * so method sharding DB_ID_RANGE can skip transactions not relevant for the DB ID range. - * + * so method sharding DB_ID_RANGE can skip transactions not relevant for the DB ID range. + * * {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME} */ private boolean txIntervalCommitTimeServiceAvailable = false; @@ -159,7 +159,7 @@ public class MetadataTracker extends AbstractShardInformationPublisher implement public MetadataTracker( Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer, boolean checkRepoServicesAvailability) { - super(true, p, client, coreName, informationServer, Tracker.Type.METADATA); + super(p, client, coreName, informationServer, Tracker.Type.METADATA); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", String.valueOf(DEFAULT_TRANSACTION_DOCS_BATCH_SIZE))); @@ -1319,6 +1319,19 @@ public class MetadataTracker extends AbstractShardInformationPublisher implement transactionsToIndex.offer(txId); } + @Override + protected void clearScheduledMaintenanceWork() + { + logAndClear(transactionsToIndex, "Transactions to be indexed"); + logAndClear(nodesToIndex, "Nodes to be indexed"); + + logAndClear(transactionsToReindex, "Transactions to be re-indexed"); + logAndClear(nodesToReindex, "Nodes to be re-indexed"); + + logAndClear(transactionsToPurge, "Transactions to be purged"); + logAndClear(nodesToPurge, "Nodes to be purged"); + } + public void addNodeToIndex(Long nodeId) { this.nodesToIndex.offer(nodeId); @@ -1333,4 +1346,6 @@ public class MetadataTracker extends AbstractShardInformationPublisher implement { this.queriesToReindex.offer(query); } + + } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 6ef37d808..4495305a0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -92,7 +92,7 @@ import org.slf4j.LoggerFactory; * deactivate ModelTracker * @enduml */ -public class ModelTracker extends AbstractTracker implements Tracker +public class ModelTracker extends AbstractTracker { private static final Logger LOGGER = LoggerFactory.getLogger(ModelTracker.class); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java deleted file mode 100644 index 822106fe1..000000000 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/NodeStatePublisher.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * #%L - * Alfresco Search Services - * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ - -package org.alfresco.solr.tracker; - -import static org.alfresco.solr.tracker.Tracker.Type.NODE_STATE_PUBLISHER; - -import org.alfresco.httpclient.AuthenticationException; -import org.alfresco.repo.index.shard.ShardState; -import org.alfresco.solr.SolrInformationServer; -import org.alfresco.solr.TrackerState; -import org.alfresco.solr.client.SOLRAPIClient; -import org.apache.commons.codec.EncoderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Semaphore; - -/** - * Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs - * Alfresco about the state of the hosting slave core. - * As the name suggests, this worker is scheduled only when the owning core acts as a slave. - * It allows Solr's master/slave setup to be used with dynamic shard registration. - * - * In this scenario the slave is polling a "tracking" Solr node. The tracker below calls - * the repo to register the state of the node without pulling any real transactions from the repo. - * - * This allows the repo to register the replica so that it will be included in queries. But the slave Solr node - * will pull its data from a "tracking" Solr node using Solr's master/slave replication, rather then tracking the repository. - * - * @author Andrea Gazzarini - * @since 1.5 - */ -public class NodeStatePublisher extends AbstractShardInformationPublisher -{ - private static final Logger LOGGER = LoggerFactory.getLogger(NodeStatePublisher.class); - - - // Share run and write locks across all SlaveCoreStatePublisher threads - private static final Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static final Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } - - public NodeStatePublisher( - boolean isMaster, - Properties coreProperties, - SOLRAPIClient repositoryClient, - String name, - SolrInformationServer informationServer) - { - super(isMaster, coreProperties, repositoryClient, name, informationServer, NODE_STATE_PUBLISHER); - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - } - - @Override - protected void doTrack(String iterationId) - { - try - { - ShardState shardstate = getShardState(); - client.getTransactions(0L, null, 0L, null, 0, shardstate); - } - catch (EncoderException | IOException | AuthenticationException exception ) - { - LOGGER.error("Unable to publish this node state. " + - "A failure condition has been met during the outbound subscription message encoding process. " + - "See the stacktrace below for further details.", exception); - } - } - - @Override - public void maintenance() - { - // Do nothing here - } - - @Override - public boolean hasMaintenance() - { - return false; - } - - /** - * When running in a slave mode, we need to recreate the tracker state every time. - * This because in that context we don't have any tracker updating the state (e.g. lastIndexedChangeSetCommitTime, - * lastIndexedChangeSetId) - * - * @return a new, fresh and up to date instance of {@link TrackerState}. - */ - @Override - public TrackerState getTrackerState() - { - return infoSrv.getTrackerInitialState(); - } -} \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractShardInformationPublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ShardStatePublisher.java similarity index 53% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractShardInformationPublisher.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ShardStatePublisher.java index ff47cdf7e..8f2a81739 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractShardInformationPublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ShardStatePublisher.java @@ -26,35 +26,31 @@ package org.alfresco.solr.tracker; -import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; -import org.alfresco.repo.dictionary.NamespaceDAO; -import org.alfresco.repo.index.shard.ShardMethodEnum; +import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.index.shard.ShardStateBuilder; -import org.alfresco.repo.search.impl.QueryParserUtils; -import org.alfresco.service.cmr.dictionary.DictionaryService; -import org.alfresco.service.cmr.dictionary.PropertyDefinition; -import org.alfresco.service.namespace.QName; import org.alfresco.solr.AlfrescoCoreAdminHandler; -import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.InformationServer; -import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.codec.EncoderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.HashMap; -import java.util.Optional; +import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import static java.util.Optional.of; -import static java.util.Optional.ofNullable; -import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_KEY_KEY; +import static org.alfresco.solr.tracker.Tracker.Type.NODE_STATE_PUBLISHER; /** - * Superclass for all components which are able to inform Alfresco about the hosting node state. + * Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs + * Alfresco about the state of the hosting core. + * * This has been introduced in SEARCH-1752 for splitting the dual responsibility of the {@link org.alfresco.solr.tracker.MetadataTracker}. * As consequence of that, this class contains only the members needed for obtaining a valid * {@link org.alfresco.repo.index.shard.ShardState} that can be periodically communicated to Alfresco. @@ -63,113 +59,80 @@ import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_KEY_KEY; * @since 1.5 * @see SEARCH-1752 */ -public abstract class AbstractShardInformationPublisher extends AbstractTracker +public class ShardStatePublisher extends AbstractTracker { - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractShardInformationPublisher.class); - DocRouter docRouter; + private static final Logger LOGGER = LoggerFactory.getLogger(ShardStatePublisher.class); + private static final Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private static final Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); + private final boolean isMaster; - /** The string representation of the shard key. */ - private Optional shardKey; - - /** The property to use for determining the shard. */ - protected Optional shardProperty = Optional.empty(); - - AbstractShardInformationPublisher( + public ShardStatePublisher( boolean isMaster, Properties p, SOLRAPIClient client, String coreName, - InformationServer informationServer, - Type type) + InformationServer informationServer) { - super(p, client, coreName, informationServer, type); + super(p, client, coreName, informationServer, NODE_STATE_PUBLISHER); + this.isMaster = isMaster; - shardKey = ofNullable(p.getProperty(SHARD_KEY_KEY)); - firstUpdateShardProperty(); - - docRouter = DocRouterFactory.getRouter(p, shardMethod); + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } - AbstractShardInformationPublisher(Type type) + @Override + protected void doTrack(String iterationId) { - super(type); - this.isMaster = false; - } - - /** - * Returns information about the {@link org.alfresco.solr.client.Node} associated with the given dbid. - * - * @param dbid the node identifier. - * @return the {@link org.alfresco.solr.client.Node} associated with the given dbid. - */ - public NodeReport checkNode(Long dbid) - { - NodeReport nodeReport = new NodeReport(); - nodeReport.setDbid(dbid); - - this.infoSrv.addCommonNodeReportInfo(nodeReport); - - return nodeReport; - } - - private void firstUpdateShardProperty() - { - shardKey.ifPresent( shardKeyName -> { - updateShardProperty(); - if (shardProperty.isEmpty()) - { - LOGGER.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); - } - }); - } - - /** - * Set the shard property using the shard key. - */ - void updateShardProperty() - { - shardKey.ifPresent(shardKeyName -> { - Optional updatedShardProperty = getShardProperty(shardKeyName); - if (!shardProperty.equals(updatedShardProperty)) - { - if (updatedShardProperty.isEmpty()) - { - LOGGER.warn("The model defining {} property has been disabled", shardKeyName); - } - else - { - LOGGER.info("New {} property found for {}", SHARD_KEY_KEY, shardKeyName); - } - } - shardProperty = updatedShardProperty; - }); - } - - /** - * Given the field name, returns the name of the property definition. - * If the property definition is not found, Empty optional is returned. - * - * @param field the field name. - * @return the name of the associated property definition if present, Optional.Empty() otherwise - */ - static Optional getShardProperty(String field) - { - if (StringUtils.isBlank(field)) + try { - throw new IllegalArgumentException("Sharding property " + SHARD_KEY_KEY + " has not been set."); + ShardState shardstate = getShardState(); + client.getTransactions(0L, null, 0L, null, 0, shardstate); } + catch (EncoderException | IOException | AuthenticationException exception ) + { + LOGGER.error("Unable to publish this node state. " + + "A failure condition has been met during the outbound subscription message encoding process. " + + "See the stacktrace below for further details.", exception); + } + } - AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance(); - NamespaceDAO namespaceDAO = dataModel.getNamespaceDAO(); - DictionaryService dictionaryService = dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT); - PropertyDefinition propertyDef = QueryParserUtils.matchPropertyDefinition("http://www.alfresco.org/model/content/1.0", - namespaceDAO, - dictionaryService, - field); + @Override + public void maintenance() + { + // Do nothing here + } - return ofNullable(propertyDef).map(PropertyDefinition::getName); + @Override + public boolean hasMaintenance() + { + return false; + } + + /** + * When running in a slave mode, we need to recreate the tracker state every time. + * This because in that context we don't have any tracker updating the state (e.g. lastIndexedChangeSetCommitTime, + * lastIndexedChangeSetId) + * + * @return a new, fresh and up to date instance of {@link TrackerState}. + */ + @Override + public TrackerState getTrackerState() + { + return infoSrv.getTrackerInitialState(); + } + + @Override + public Semaphore getWriteLock() + { + return WRITE_LOCK_BY_CORE.get(coreName); + } + + @Override + public Semaphore getRunLock() + { + return RUN_LOCK_BY_CORE.get(coreName); } /** @@ -177,7 +140,6 @@ public abstract class AbstractShardInformationPublisher extends AbstractTracker * {@link MetadataTracker} instance. * * @return the {@link ShardState} instance which stores the current state of the hosting shard. - * @see NodeStatePublisher */ ShardState getShardState() { @@ -222,16 +184,6 @@ public abstract class AbstractShardInformationPublisher extends AbstractTracker } - /** - * Returns the {@link DocRouter} instance in use on this node. - * - * @return the {@link DocRouter} instance in use on this node. - */ - public DocRouter getDocRouter() - { - return this.docRouter; - } - /** * Returns true if the hosting core is master or standalone. * diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java index 17a557019..0e95205b8 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java @@ -148,10 +148,19 @@ public abstract class Utils */ public static boolean isNullOrEmpty(String value) { - return ofNullable(value) - .map(String::trim) - .map(String::isEmpty) - .orElse(true); + return value == null || value.trim().length() == 0; + } + + /** + * Returns true if the input string is not null and it is not empty. + * Note whitespaces are not considered, so if a string contains only whitespaces, it is considered empty. + * + * @param value the input string. + * @return true if the input string is not null and it is not empty. + */ + public static boolean isNotNullAndNotEmpty(String value) + { + return value != null && value.trim().length() != 0; } /** diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java index 27572311f..d73d2638b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java @@ -191,6 +191,16 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS h.reload(); } + protected void disableIndexing() + { + admin.actionDisableIndexing(new ModifiableSolrParams()); + } + + protected void enableIndexing() + { + admin.actionEnableIndexing(new ModifiableSolrParams()); + } + /** * @deprecated as testHarness is used * Get admin core handler diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java index 33587101c..d380eeeb2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java @@ -62,7 +62,7 @@ import org.alfresco.solr.tracker.DocRouter; import org.alfresco.solr.tracker.IndexHealthReport; import org.alfresco.solr.tracker.MetadataTracker; import org.alfresco.solr.tracker.PropertyRouter; -import org.alfresco.solr.tracker.NodeStatePublisher; +import org.alfresco.solr.tracker.ShardStatePublisher; import org.alfresco.solr.tracker.TrackerRegistry; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; @@ -219,23 +219,22 @@ public class AlfrescoCoreAdminHandlerIT } @Test - public void coreIsMaster_thenCoreStatePublisherInstanceCorrespondsToMetadataTracker() + public void coreIsMaster_thenCoreStatePublisherInstanceCorrespondsToShardStatePublisher() { - MetadataTracker coreStatePublisher = mock(MetadataTracker.class); + ShardStatePublisher coreStatePublisher = mock(ShardStatePublisher.class); - when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))) + when(trackerRegistry.getTrackerForCore(anyString(), eq(ShardStatePublisher.class))) .thenReturn(coreStatePublisher); assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); } @Test - public void coreIsSlave_thenCoreStatePublisherInstanceCorrespondsToSlaveCoreStatePublisher() + public void coreIsSlave_thenCoreStatePublisherInstanceCorrespondsToShardStatePublisher() { - NodeStatePublisher coreStateTracker = mock(NodeStatePublisher.class); + ShardStatePublisher coreStateTracker = mock(ShardStatePublisher.class); - when(trackerRegistry.getTrackerForCore(anyString(), eq(MetadataTracker.class))).thenReturn(null); - when(trackerRegistry.getTrackerForCore(anyString(), eq(NodeStatePublisher.class))).thenReturn(coreStateTracker); + when(trackerRegistry.getTrackerForCore(anyString(), eq(ShardStatePublisher.class))).thenReturn(coreStateTracker); assertSame(coreStateTracker, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java index 3a8c7ef8f..52f5b67a2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java @@ -26,37 +26,21 @@ package org.alfresco.solr; -import org.alfresco.solr.adapters.IOpenBitSet; -import org.alfresco.solr.adapters.SolrOpenBitSetAdapter; -import org.alfresco.solr.tracker.AclTracker; -import org.alfresco.solr.tracker.IndexHealthReport; -import org.alfresco.solr.tracker.MetadataTracker; -import org.alfresco.solr.tracker.TrackerRegistry; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrResourceLoader; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - import static java.util.Optional.of; import static java.util.stream.IntStream.range; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACL_TX_IN_INDEX_NOT_IN_DB; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_ERROR_MESSAGE_LABEL; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_LABEL; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_NOT_SCHEDULED; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_SCHEDULED; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ADDITIONAL_INFO; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ALFRESCO_CORE_NAME; import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARCHIVE_CORE_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_ACLID; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_ACLTXID; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_NODEID; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARG_TXID; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.CORE_PARAMETER_NAMES; import static org.alfresco.solr.AlfrescoCoreAdminHandler.DRY_RUN_PARAMETER_NAME; import static org.alfresco.solr.AlfrescoCoreAdminHandler.DUPLICATED_ACL_TX_IN_INDEX; import static org.alfresco.solr.AlfrescoCoreAdminHandler.DUPLICATED_TX_IN_INDEX; @@ -69,13 +53,53 @@ import static org.alfresco.solr.AlfrescoCoreAdminHandler.TO_TX_COMMIT_TIME_PARAM import static org.alfresco.solr.AlfrescoCoreAdminHandler.TX_IN_INDEX_NOT_IN_DB; import static org.alfresco.solr.AlfrescoCoreAdminHandler.UNKNOWN_CORE_MESSAGE; import static org.alfresco.solr.AlfrescoCoreAdminHandler.UNPROCESSABLE_REQUEST_ON_SLAVE_NODES; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.VERSION_CORE_NAME; +import static org.apache.solr.common.params.CoreAdminParams.ACTION; import static org.apache.solr.common.params.CoreAdminParams.CORE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import org.alfresco.solr.adapters.IOpenBitSet; +import org.alfresco.solr.adapters.SolrOpenBitSetAdapter; +import org.alfresco.solr.client.SOLRAPIClient; +import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.IndexHealthReport; +import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.TrackerRegistry; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.json.JSONException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + @RunWith(MockitoJUnitRunner.class) public class AlfrescoCoreAdminHandlerTest { @@ -97,7 +121,7 @@ public class AlfrescoCoreAdminHandlerTest } @Test - public void noTargetCoreInParams() + public void noTargetCoreToFixInParams() { assertEquals(0, params.size()); @@ -106,7 +130,7 @@ public class AlfrescoCoreAdminHandlerTest } @Test - public void unknownTargetCoreInParams() + public void unknownTargetCoreToFixInParams() { String invalidCoreName = "thisIsAnInvalidOrAtLeastUnknownCoreName"; params.set(CORE, invalidCoreName); @@ -213,6 +237,86 @@ public class AlfrescoCoreAdminHandlerTest assertEquals(ACTION_STATUS_NOT_SCHEDULED, actionResponse.get(ACTION_STATUS_LABEL)); } + @Test + public void masterOrStandaloneNodeWithTrackersDisabled_DryRunParameterShouldBeForcedToTrue() + { + class TestMetadataTracker extends MetadataTracker { + + protected TestMetadataTracker() { + super(new Properties(), mock(SOLRAPIClient.class), ALFRESCO_CORE_NAME, mock(InformationServer.class)); + this.state = new TrackerState(); + } + + @Override + protected void doTrack(String iterationId) { + // Nothing to be done here, it's a fake implementation. + } + + @Override + protected void setPersistentIndexingStateAcrossReloadsTo(boolean enabled) { + // Nothing to be done here, it's a fake implementation. + } + } + + class TestAclTracker extends AclTracker { + + protected TestAclTracker() { + super(new Properties(), mock(SOLRAPIClient.class), ALFRESCO_CORE_NAME, mock(InformationServer.class)); + this.state = new TrackerState(); + } + + @Override + protected void doTrack(String iterationId) { + // Nothing to be done here, it's a fake implementation. + } + + @Override + protected void setPersistentIndexingStateAcrossReloadsTo(boolean enabled) { + // Nothing to be done here, it's a fake implementation. + } + } + + admin = new AlfrescoCoreAdminHandler() { + @Override + NamedList fixOnSpecificCore( + String coreName, + Long fromTxCommitTime, + Long toTxCommitTime, + boolean dryRun, + int maxTransactionsToSchedule) { + return new NamedList<>(); // dummy entry + } + + @Override + boolean isMasterOrStandalone(String coreName) + { + return true; + } + }; + + admin.trackerRegistry = registry; + final MetadataTracker metadataTracker = new TestMetadataTracker(); + final AclTracker aclTracker = new TestAclTracker(); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + when(registry.getTrackersForCore(ALFRESCO_CORE_NAME)).thenReturn(List.of(metadataTracker, aclTracker)); + + params.set(CORE, ALFRESCO_CORE_NAME); + + // Let's disable tracking on Alfresco + admin.actionDisableIndexing(params); + + params.set(DRY_RUN_PARAMETER_NAME, false); + + NamedList actionResponse = admin.actionFIX(params); + assertEquals(false, actionResponse.get(DRY_RUN_PARAMETER_NAME)); + assertEquals(ACTION_STATUS_NOT_SCHEDULED, actionResponse.get(ACTION_STATUS_LABEL)); + assertNotNull( + "There should be a message which informs the requestor about the actual dryRun execution", + actionResponse.get(ADDITIONAL_INFO)); + } + @Test public void masterOrStandaloneNode_explicitDryRunParameterIsEchoed() { @@ -482,6 +586,532 @@ public class AlfrescoCoreAdminHandlerTest .orElseThrow(() -> new RuntimeException(MISSING_ACL_TX_IN_INDEX + " section not found in response."))); } + @Test + public void disableIndexingActionParameter_shouldTriggerTheDisableIndexingAction() + { + final AtomicBoolean invocationMarker = new AtomicBoolean(); + admin = new AlfrescoCoreAdminHandler() { + @Override + NamedList fixOnSpecificCore( + String coreName, + Long fromTxCommitTime, + Long toTxCommitTime, + boolean dryRun, + int maxTransactionsToSchedule) { + return new NamedList<>(); // dummy entry + } + + @Override + NamedList actionDisableIndexing(SolrParams params) throws JSONException { + invocationMarker.set(true); + return new SimpleOrderedMap<>(); + } + }; + + params.set(ACTION, "DISABLE-INDEXING"); + + SolrQueryRequest request = mock(SolrQueryRequest.class); + when(request.getParams()).thenReturn(params); + + admin.handleCustomAction(request, mock(SolrQueryResponse.class)); + + assertTrue(invocationMarker.get()); + } + + @Test + public void enableIndexingActionParameter_shouldTriggerTheIndexingEnabling() + { + final AtomicBoolean invocationMarker = new AtomicBoolean(); + admin = new AlfrescoCoreAdminHandler() { + @Override + NamedList fixOnSpecificCore( + String coreName, + Long fromTxCommitTime, + Long toTxCommitTime, + boolean dryRun, + int maxTransactionsToSchedule) { + return new NamedList<>(); // dummy entry + } + + @Override + NamedList actionEnableIndexing(SolrParams params) throws JSONException { + invocationMarker.set(true); + return new SimpleOrderedMap<>(); + } + }; + + params.set(ACTION, "ENABLE-INDEXING"); + + SolrQueryRequest request = mock(SolrQueryRequest.class); + when(request.getParams()).thenReturn(params); + + admin.handleCustomAction(request, mock(SolrQueryResponse.class)); + + assertTrue(invocationMarker.get()); + } + + @Test + public void unknownCoreNameInDisableIndexingCommand_shouldReturnAnErrorResponse() + { + String unknownCoreName = "ThisShouldBeAnInexistentCore"; + CORE_PARAMETER_NAMES.forEach(parameter -> { + params.set(parameter, unknownCoreName); + + NamedList response = admin.actionDisableIndexing(params); + assertEquals(UNKNOWN_CORE_MESSAGE + unknownCoreName, response.get(ACTION_ERROR_MESSAGE_LABEL)); + }); + } + + @Test + public void unknownCoreNameInEnableIndexingCommand_shouldReturnAnErrorResponse() + { + String unknownCoreName = "ThisShouldBeAnInexistentCore"; + CORE_PARAMETER_NAMES.forEach(parameter -> { + params.set(parameter, unknownCoreName); + + NamedList response = admin.actionEnableIndexing(params); + assertEquals(UNKNOWN_CORE_MESSAGE + unknownCoreName, response.get(ACTION_ERROR_MESSAGE_LABEL)); + }); + } + + @Test + public void disableIndexingOnSpecificSlaveCore_shouldReturnAnErrorResponse() + { + // The admin handler detects if a core is slave, master or standalone by checking + // the trackers installed on it. If no trackers have been registered, then the core is considered a slave. + assertFalse(admin.isMasterOrStandalone(ALFRESCO_CORE_NAME)); + CORE_PARAMETER_NAMES.forEach(parameter -> { + params.set(parameter, ALFRESCO_CORE_NAME); + + NamedList response = admin.actionDisableIndexing(params); + assertEquals(UNPROCESSABLE_REQUEST_ON_SLAVE_NODES, response.get(ACTION_ERROR_MESSAGE_LABEL)); + }); + } + + @Test + public void enableIndexingOnSpecificSlaveCore_shouldReturnAnErrorResponse() + { + // The admin handler detects if a core is slave, master or standalone by checking + // the trackers installed on it. If no trackers have been registered, then the core is considered a slave. + assertFalse(admin.isMasterOrStandalone(ALFRESCO_CORE_NAME)); + CORE_PARAMETER_NAMES.forEach(parameter -> { + params.set(parameter, ALFRESCO_CORE_NAME); + + NamedList response = admin.actionEnableIndexing(params); + assertEquals(UNPROCESSABLE_REQUEST_ON_SLAVE_NODES, response.get(ACTION_ERROR_MESSAGE_LABEL)); + }); + } + + @Test + public void disableIndexingWithoutIndicatingSpecificCore_shouldHaveNoEffectIfAllCoresAreSlave() + { + admin = spy(new AlfrescoCoreAdminHandler()); + admin.trackerRegistry = registry; + when(registry.getCoreNames()).thenReturn(Set.of(ALFRESCO_CORE_NAME, ARCHIVE_CORE_NAME)); + admin.actionDisableIndexing(params); + + verify(admin, times(0)).disableIndexingOnSpecificCore(anyString()); + } + + @Test + public void enableIndexingWithoutIndicatingSpecificCore_shouldHaveNoEffectIfAllCoresAreSlave() + { + admin = spy(new AlfrescoCoreAdminHandler()); + admin.trackerRegistry = registry; + when(registry.getCoreNames()).thenReturn(Set.of(ALFRESCO_CORE_NAME, ARCHIVE_CORE_NAME)); + admin.actionEnableIndexing(params); + + verify(admin, times(0)).enableIndexingOnSpecificCore(anyString()); + } + + @Test + public void disableIndexingWithoutIndicatingSpecificCore_shouldAffectOnlyMasterOrStandaloneCores() + { + admin = spy(new AlfrescoCoreAdminHandler()); + admin.trackerRegistry = registry; + when(registry.getCoreNames()).thenReturn(Set.of(ALFRESCO_CORE_NAME, ARCHIVE_CORE_NAME, VERSION_CORE_NAME)); + + // "alfresco" and "archive" are master/standalone cores, "version" is a slave core + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(mock(MetadataTracker.class)); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(mock(MetadataTracker.class)); + when(registry.getTrackerForCore(VERSION_CORE_NAME, MetadataTracker.class)).thenReturn(null); + + admin.actionDisableIndexing(params); + + verify(admin, times(1)).disableIndexingOnSpecificCore(ALFRESCO_CORE_NAME); + verify(admin, times(1)).disableIndexingOnSpecificCore(ARCHIVE_CORE_NAME); + } + + @Test + public void enableIndexingWithoutIndicatingSpecificCore_shouldAffectOnlyMasterOrStandaloneCores() + { + admin = spy(new AlfrescoCoreAdminHandler()); + admin.trackerRegistry = registry; + when(registry.getCoreNames()).thenReturn(Set.of(ALFRESCO_CORE_NAME, ARCHIVE_CORE_NAME, VERSION_CORE_NAME)); + + // "alfresco" and "archive" are master/standalone cores, "version" is a slave core + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(mock(MetadataTracker.class)); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(mock(MetadataTracker.class)); + when(registry.getTrackerForCore(VERSION_CORE_NAME, MetadataTracker.class)).thenReturn(null); + + admin.actionEnableIndexing(params); + + verify(admin, times(1)).enableIndexingOnSpecificCore(ALFRESCO_CORE_NAME); + verify(admin, times(1)).enableIndexingOnSpecificCore(ARCHIVE_CORE_NAME); + } + + @Test + public void retryActionOnSlaveNode_shouldReturnWarningMessage() + { + admin.coreNames().forEach(coreName -> assertFalse(admin.isMasterOrStandalone(coreName))); + + NamedList actionResponse = admin.actionRETRY(params); + assertNotNull(actionResponse.get(AlfrescoCoreAdminHandler.WARNING)); + } + + @Test + public void retryActionWhenIndexingIsDisabled_shouldReturnAnInfoMessage() + { + // That is not true: each core has an its own InformationServer instance + // However for this specific test we don't care + InformationServer srv = mock(InformationServer.class); + admin.informationServers = new ConcurrentHashMap<>(); + admin.informationServers.put(ALFRESCO_CORE_NAME, srv); + admin.informationServers.put(ARCHIVE_CORE_NAME, srv); + + // That is not true: each core has an its own MetadataTracker instance + // However for this specific test we don't care + MetadataTracker metadataTracker = mock(MetadataTracker.class); + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + + when(metadataTracker.isEnabled()).thenReturn(false); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionRETRY(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_NOT_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + assertNotNull(response.get(ADDITIONAL_INFO)); + }); + + verifyNoInteractions(srv); + } + + @Test + public void retryActionWhenIndexingIsEnabled_shouldCollectThingsToReindex() throws Exception + { + final Set alfrescoErrorNodeIds = Set.of(123452L, 13579L, 24680L, 98765L); + final Set archiveErrorNodeIds = Set.of(1234520L, 913579L, 124680L, 598765L); + + InformationServer alfrescoInformationServer = mock(InformationServer.class); + InformationServer archiveInformationServer = mock(InformationServer.class); + + admin.informationServers = new ConcurrentHashMap<>(); + admin.informationServers.put(ALFRESCO_CORE_NAME, alfrescoInformationServer); + admin.informationServers.put(ARCHIVE_CORE_NAME, archiveInformationServer); + + when(alfrescoInformationServer.getErrorDocIds()).thenReturn(alfrescoErrorNodeIds); + when(archiveInformationServer.getErrorDocIds()).thenReturn(archiveErrorNodeIds); + + MetadataTracker alfrescoMetadataTracker = mock(MetadataTracker.class); + MetadataTracker archiveMetadataTracker = mock(MetadataTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(alfrescoMetadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(archiveMetadataTracker); + + when(alfrescoMetadataTracker.isEnabled()).thenReturn(true); + when(archiveMetadataTracker.isEnabled()).thenReturn(true); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionRETRY(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + }); + + alfrescoErrorNodeIds.forEach(id -> verify(alfrescoMetadataTracker).addNodeToReindex(id)); + archiveErrorNodeIds.forEach(id -> verify(archiveMetadataTracker).addNodeToReindex(id)); + } + + @Test + public void indexActionOnSlaveNode_shouldReturnWarningMessage() + { + admin.coreNames().forEach(coreName -> assertFalse(admin.isMasterOrStandalone(coreName))); + + NamedList actionResponse = admin.actionINDEX(params); + assertNotNull(actionResponse.get(AlfrescoCoreAdminHandler.WARNING)); + } + + @Test + public void indexActionWhenIndexingIsDisabled_shouldReturnAnInfoMessage() + { + MetadataTracker metadataTracker = mock(MetadataTracker.class); + AclTracker aclTracker = mock(AclTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + + when(metadataTracker.isEnabled()).thenReturn(false); + when(aclTracker.isEnabled()).thenReturn(false); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionINDEX(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_NOT_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + assertNotNull(response.get(ADDITIONAL_INFO)); + }); + } + + @Test + public void indexActionWhenIndexingIsEnabled_shouldCollectThingsToReindex() + { + final String txIdParam = "123452"; + final String aclTxIdParam = "13579"; + final String nodeIdParam = "24680"; + final String aclIdParam = "98765"; + + params.set(ARG_TXID, txIdParam); + params.set(ARG_ACLTXID, aclTxIdParam); + params.set(ARG_NODEID, nodeIdParam); + params.set(ARG_ACLID, aclIdParam); + + MetadataTracker alfrescoMetadataTracker = mock(MetadataTracker.class); + AclTracker alfrescoAclTracker = mock(AclTracker.class); + + MetadataTracker archiveMetadataTracker = mock(MetadataTracker.class); + AclTracker archiveAclTracker = mock(AclTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(alfrescoMetadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(archiveMetadataTracker); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(alfrescoAclTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, AclTracker.class)).thenReturn(archiveAclTracker); + + when(alfrescoMetadataTracker.isEnabled()).thenReturn(true); + when(alfrescoAclTracker.isEnabled()).thenReturn(true); + + when(archiveMetadataTracker.isEnabled()).thenReturn(true); + when(archiveAclTracker.isEnabled()).thenReturn(true); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionINDEX(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + }); + + verify(alfrescoMetadataTracker).addTransactionToIndex(Long.parseLong(txIdParam)); + verify(alfrescoMetadataTracker).addNodeToIndex(Long.parseLong(nodeIdParam)); + verify(alfrescoAclTracker).addAclChangeSetToIndex(Long.parseLong(aclTxIdParam)); + verify(alfrescoAclTracker).addAclToIndex(Long.parseLong(aclIdParam)); + + verify(archiveMetadataTracker).addTransactionToIndex(Long.parseLong(txIdParam)); + verify(archiveMetadataTracker).addNodeToIndex(Long.parseLong(nodeIdParam)); + verify(archiveAclTracker).addAclChangeSetToIndex(Long.parseLong(aclTxIdParam)); + verify(archiveAclTracker).addAclToIndex(Long.parseLong(aclIdParam)); + } + + @Test + public void reindexActionOnSlaveNode_shouldReturnWarningMessage() + { + admin.coreNames().forEach(coreName -> assertFalse(admin.isMasterOrStandalone(coreName))); + + NamedList actionResponse = admin.actionREINDEX(params); + assertNotNull(actionResponse.get(AlfrescoCoreAdminHandler.WARNING)); + } + + @Test + public void reindexActionWhenIndexingIsDisabled_shouldReturnAnInfoMessage() + { + MetadataTracker metadataTracker = mock(MetadataTracker.class); + AclTracker aclTracker = mock(AclTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + + when(metadataTracker.isEnabled()).thenReturn(false); + when(aclTracker.isEnabled()).thenReturn(false); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionREINDEX(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_NOT_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + assertNotNull(response.get(ADDITIONAL_INFO)); + }); + } + + @Test + public void reindexActionWhenIndexingIsEnabled_shouldCollectThingsToReindex() + { + final String txIdParam = "123452"; + final String aclTxIdParam = "13579"; + final String nodeIdParam = "24680"; + final String aclIdParam = "98765"; + + params.set(ARG_TXID, txIdParam); + params.set(ARG_ACLTXID, aclTxIdParam); + params.set(ARG_NODEID, nodeIdParam); + params.set(ARG_ACLID, aclIdParam); + + MetadataTracker alfrescoMetadataTracker = mock(MetadataTracker.class); + AclTracker alfrescoAclTracker = mock(AclTracker.class); + + MetadataTracker archiveMetadataTracker = mock(MetadataTracker.class); + AclTracker archiveAclTracker = mock(AclTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(alfrescoMetadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(archiveMetadataTracker); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(alfrescoAclTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, AclTracker.class)).thenReturn(archiveAclTracker); + + when(alfrescoMetadataTracker.isEnabled()).thenReturn(true); + when(alfrescoAclTracker.isEnabled()).thenReturn(true); + + when(archiveMetadataTracker.isEnabled()).thenReturn(true); + when(archiveAclTracker.isEnabled()).thenReturn(true); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionREINDEX(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + }); + + verify(alfrescoMetadataTracker).addTransactionToReindex(Long.parseLong(txIdParam)); + verify(alfrescoMetadataTracker).addNodeToReindex(Long.parseLong(nodeIdParam)); + verify(alfrescoAclTracker).addAclChangeSetToReindex(Long.parseLong(aclTxIdParam)); + verify(alfrescoAclTracker).addAclToReindex(Long.parseLong(aclIdParam)); + + verify(archiveMetadataTracker).addTransactionToReindex(Long.parseLong(txIdParam)); + verify(archiveMetadataTracker).addNodeToReindex(Long.parseLong(nodeIdParam)); + verify(archiveAclTracker).addAclChangeSetToReindex(Long.parseLong(aclTxIdParam)); + verify(archiveAclTracker).addAclToReindex(Long.parseLong(aclIdParam)); + } + + @Test + public void purgeActionOnSlaveNode_shouldReturnWarningMessage() + { + admin.coreNames().forEach(coreName -> assertFalse(admin.isMasterOrStandalone(coreName))); + + NamedList actionResponse = admin.actionPURGE(params); + assertNotNull(actionResponse.get(AlfrescoCoreAdminHandler.WARNING)); + } + + @Test + public void purgeActionWhenIndexingIsDisabled_shouldReturnAnInfoMessage() + { + MetadataTracker metadataTracker = mock(MetadataTracker.class); + AclTracker aclTracker = mock(AclTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, AclTracker.class)).thenReturn(aclTracker); + + when(metadataTracker.isEnabled()).thenReturn(false); + when(aclTracker.isEnabled()).thenReturn(false); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionPURGE(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_NOT_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + assertNotNull(response.get(ADDITIONAL_INFO)); + }); + } + + @Test + public void purgeActionWhenIndexingIsEnabled_shouldCollectTransactionsToPurge() + { + final String txIdParam = "123452"; + final String aclTxIdParam = "13579"; + final String nodeIdParam = "24680"; + final String aclIdParam = "98765"; + + params.set(ARG_TXID, txIdParam); + params.set(ARG_ACLTXID, aclTxIdParam); + params.set(ARG_NODEID, nodeIdParam); + params.set(ARG_ACLID, aclIdParam); + + MetadataTracker alfrescoMetadataTracker = mock(MetadataTracker.class); + AclTracker alfrescoAclTracker = mock(AclTracker.class); + + MetadataTracker archiveMetadataTracker = mock(MetadataTracker.class); + AclTracker archiveAclTracker = mock(AclTracker.class); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, MetadataTracker.class)).thenReturn(alfrescoMetadataTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, MetadataTracker.class)).thenReturn(archiveMetadataTracker); + + when(registry.getTrackerForCore(ALFRESCO_CORE_NAME, AclTracker.class)).thenReturn(alfrescoAclTracker); + when(registry.getTrackerForCore(ARCHIVE_CORE_NAME, AclTracker.class)).thenReturn(archiveAclTracker); + + when(alfrescoMetadataTracker.isEnabled()).thenReturn(true); + when(alfrescoAclTracker.isEnabled()).thenReturn(true); + + when(archiveMetadataTracker.isEnabled()).thenReturn(true); + when(archiveAclTracker.isEnabled()).thenReturn(true); + + admin.coreNames().forEach(coreName -> assertTrue(admin.isMasterOrStandalone(coreName))); + + final NamedList actionResponse = admin.actionPURGE(params); + admin.coreNames() + .stream() + .map(actionResponse::get) + .map(NamedList.class::cast) + .forEach(response -> { + assertEquals(ACTION_STATUS_SCHEDULED, response.get(ACTION_STATUS_LABEL)); + }); + + verify(alfrescoMetadataTracker).addTransactionToPurge(Long.parseLong(txIdParam)); + verify(alfrescoMetadataTracker).addNodeToPurge(Long.parseLong(nodeIdParam)); + verify(alfrescoAclTracker).addAclChangeSetToPurge(Long.parseLong(aclTxIdParam)); + verify(alfrescoAclTracker).addAclToPurge(Long.parseLong(aclIdParam)); + + verify(archiveMetadataTracker).addTransactionToPurge(Long.parseLong(txIdParam)); + verify(archiveMetadataTracker).addNodeToPurge(Long.parseLong(nodeIdParam)); + verify(archiveAclTracker).addAclChangeSetToPurge(Long.parseLong(aclTxIdParam)); + verify(archiveAclTracker).addAclToPurge(Long.parseLong(aclIdParam)); + } + private void assertThatExplicitParameterIsEchoed(String parameterName, T parameterValue) { admin = new AlfrescoCoreAdminHandler() { diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoIndexingStatePersistenceAcrossReloadsIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoIndexingStatePersistenceAcrossReloadsIT.java new file mode 100644 index 000000000..6313c04b6 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoIndexingStatePersistenceAcrossReloadsIT.java @@ -0,0 +1,180 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr; + +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.SOLRAPIQueueClient; +import org.alfresco.solr.client.Transaction; +import org.alfresco.solr.tracker.ActivatableTracker; +import org.alfresco.solr.tracker.Tracker; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.quartz.SchedulerException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.util.Collections.singletonList; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@SolrTestCaseJ4.SuppressSSL +public class AlfrescoIndexingStatePersistenceAcrossReloadsIT extends AbstractAlfrescoSolrIT +{ + @BeforeClass + public static void beforeClass() throws Exception + { + initAlfrescoCore("schema.xml"); + admin = (AlfrescoCoreAdminHandler)getCore().getCoreContainer().getMultiCoreHandler(); + } + + @After + public void clearQueue() + { + SOLRAPIQueueClient.NODE_META_DATA_MAP.clear(); + SOLRAPIQueueClient.TRANSACTION_QUEUE.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); + SOLRAPIQueueClient.NODE_MAP.clear(); + } + + @Test + public void testIndexingStateAcrossReloads() throws Exception + { + long localId = 0L; + + AclChangeSet aclChangeSet = getAclChangeSet(1, ++localId); + + Acl acl = getAcl(aclChangeSet); + + AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + + indexAclChangeSet(aclChangeSet, + singletonList(acl), + singletonList(aclReaders)); + + int numNodes = 1; + List nodes = new ArrayList<>(); + List nodeMetaDatas = new ArrayList<>(); + + Transaction bigTxn = getTransaction(0, numNodes, ++localId); + + for(int i=0; i trackers = getTrackers(); + + disableIndexing(); + + // Make sure trackers have been disabled + Collection activatableTrackers = + getTrackers().stream() + .filter(tracker -> tracker instanceof ActivatableTracker) + .map(ActivatableTracker.class::cast) + .collect(Collectors.toList()); + + assertFalse(activatableTrackers.isEmpty()); + activatableTrackers.forEach(tracker -> assertTrue(tracker.isDisabled())); + + // Reload the core + reloadAndAssertCorrect(trackers, trackers.size(), getJobsCount()); + + // Make sure indexing is disabled in the reloaded core + Collection activatableTrackersBelongingToReloadedCore = + getTrackers().stream() + .filter(tracker -> tracker instanceof ActivatableTracker) + .map(ActivatableTracker.class::cast) + .collect(Collectors.toList()); + + assertFalse(activatableTrackersBelongingToReloadedCore.isEmpty()); + activatableTrackersBelongingToReloadedCore.forEach(tracker -> assertTrue(tracker.isDisabled())); + + // Re-enable indexing + enableIndexing(); + + // Make sure tracking has been enabled + activatableTrackersBelongingToReloadedCore.forEach(tracker -> assertTrue(tracker.isEnabled())); + + Transaction bigTxn2 = getTransaction(0, numNodes, ++localId); + for(int i=0; i trackers, int numOfTrackers, int jobs) throws Exception + { + reload(); + //Give it a little time to shutdown properly and recover. + TimeUnit.SECONDS.sleep(1); + + Collection reloadedTrackers = getTrackers(); + assertEquals("After a reload the number of trackers should be the same", numOfTrackers, getTrackers().size()); + assertEquals("After a reload the number of jobs should be the same", jobs, getJobsCount()); + + trackers.forEach(tracker -> assertFalse("The reloaded trackers should be different.", reloadedTrackers.contains(tracker))); + } + + private int getJobsCount() throws SchedulerException + { + return admin.getScheduler().getJobsCount(); + } +} \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ActivatableTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ActivatableTrackerTest.java new file mode 100644 index 000000000..fb42b65de --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ActivatableTrackerTest.java @@ -0,0 +1,191 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.solr.tracker; + +import org.alfresco.solr.InformationServer; +import org.alfresco.solr.TrackerState; +import org.alfresco.solr.client.SOLRAPIClient; +import org.junit.Before; +import org.junit.Test; + +import java.util.Properties; +import java.util.concurrent.Semaphore; +import java.util.stream.Stream; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +public class ActivatableTrackerTest +{ + private static class TestActivatableTracker extends ActivatableTracker { + + protected TestActivatableTracker(Properties properties, TrackerState state) { + super(properties, mock(SOLRAPIClient.class), "thisIsTheCoreName", mock(InformationServer.class), Type.NODE_STATE_PUBLISHER); + this.state = state; + } + + @Override + protected void doTrack(String iterationId) { + // Nothing to be done here, it's a fake implementation. + } + + @Override + protected void setPersistentIndexingStateAcrossReloadsTo(boolean enabled) { + // Nothing to be done here, it's a fake implementation. + } + + @Override + public void maintenance() { + + } + + @Override + public boolean hasMaintenance() { + return false; + } + + @Override + public Semaphore getWriteLock() { + return null; + } + + @Override + public Semaphore getRunLock() { + return null; + } + } + + private ActivatableTracker tracker; + private TrackerState state; + + @Before + public void setUp() + { + state = new TrackerState(); + state.setRunning(false); + tracker = spy(new TestActivatableTracker(new Properties(), state)); + assertFalse(tracker.state.isRunning()); + } + + @Test + public void enabledShouldBeTheDefaultState() + { + assertTrue(tracker.isEnabled()); + } + + @Test + public void trackersCanBeExplicitlyDisabled() + { + assertTrue(tracker.isEnabled()); + + tracker.disable(); + + assertFalse(tracker.isEnabled()); + } + + @Test + public void disablingATracker_shouldClearTheScheduledMaintenanceWork() + { + assertTrue(tracker.isEnabled()); + + tracker.disable(); + + assertFalse(tracker.isEnabled()); + verify(tracker).clearScheduledMaintenanceWork(); + } + + @Test + public void enableIsIdempotent() + { + assertTrue(tracker.isEnabled()); + + tracker.enable(); + + assertTrue(tracker.isEnabled()); + + tracker.disable(); + + assertFalse(tracker.isEnabled()); + + tracker.enable(); + assertTrue(tracker.isEnabled()); + + tracker.enable(); + assertTrue(tracker.isEnabled()); + } + + @Test + public void disableIsIdempotent() + { + assertTrue(tracker.isEnabled()); + + tracker.disable(); + assertFalse(tracker.isEnabled()); + + tracker.disable(); + assertFalse(tracker.isEnabled()); + + tracker.enable(); + assertTrue(tracker.isEnabled()); + + tracker.disable(); + assertFalse(tracker.isEnabled()); + + tracker.disable(); + assertFalse(tracker.isEnabled()); + } + + @Test + public void disableIndexingOnRunningTracker_shouldDisableTheTrackerAnSetItInRollbackMode() + { + state.setRunning(true); + assertTrue(tracker.isEnabled()); + assertTrue(tracker.state.isRunning()); + + tracker.disable(); + + state.setRunning(true); + assertTrue(tracker.state.isRunning()); + + assertFalse(tracker.isEnabled()); + verify(tracker).setRollback(true, null); + } + + @Test + public void assertActivatableTrackersList() { + Stream.of(MetadataTracker.class, AclTracker.class, ContentTracker.class, CascadeTracker.class) + .forEach(clazz -> assertTrue("Warning: " + clazz + " is supposed to be enabled/disabled", ActivatableTracker.class.isAssignableFrom(clazz))); + } + + @Test + public void assertAlwaysActivatedTrackersList() { + Stream.of(CommitTracker.class, ModelTracker.class) + .forEach(clazz -> assertFalse("Warning: " + clazz + " is not supposed to be enabled/disabled", ActivatableTracker.class.isAssignableFrom(clazz))); + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java index f86c86363..c267815fc 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerStateIT.java @@ -71,7 +71,7 @@ import java.util.stream.Collectors; /** * A partial state of {@link org.alfresco.solr.TrackerState} is exposed through two interfaces: AdminHandler.SUMMARY and - * {@link MetadataTracker#getShardState}. + * {@link ShardStatePublisher#getShardState}. * * This test makes sure that state is consistent across the two mentioned approaches. That is, properties returned by the * Core SUMMARY must have the same value of the same properties in the ShardState. @@ -109,10 +109,10 @@ public class AlfrescoSolrTrackerStateIT extends AbstractAlfrescoSolrIT public void shardStateMustBeConsistentWithCoreSummaryStats() throws Exception { SolrCore core = getCore(); - MetadataTracker tracker = + ShardStatePublisher tracker = of(coreAdminHandler(core)) .map(AlfrescoCoreAdminHandler::getTrackerRegistry) - .map(registry -> registry.getTrackerForCore(core.getName(), MetadataTracker.class)) + .map(registry -> registry.getTrackerForCore(core.getName(), ShardStatePublisher.class)) .orElseThrow(() -> new IllegalStateException("Cannot retrieve the Metadata tracker on this test core.")); // 1. First consistency check: ShardState must have the same values of CoreAdmin.SUMMARY report diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java index 0f44d50a4..8d5b2c01d 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerStateIT.java @@ -67,7 +67,7 @@ import java.util.stream.Collectors; /** * A partial state of {@link org.alfresco.solr.TrackerState} is exposed through two interfaces: AdminHandler.SUMMARY and - * {@link MetadataTracker#getShardState}. + * {@link ShardStatePublisher#getShardState}. * This test makes sure that state is consistent across the two mentioned approaches. That is, properties returned by the * Core SUMMARY must have the same value of the same properties in the ShardState. * @@ -99,10 +99,10 @@ public class DistributedAlfrescoSolrTrackerStateIT extends AbstractAlfrescoDistr putHandleDefaults(); getCores(solrShards).forEach(core -> { - MetadataTracker tracker = + ShardStatePublisher tracker = of(coreAdminHandler(core)) .map(AlfrescoCoreAdminHandler::getTrackerRegistry) - .map(registry -> registry.getTrackerForCore(core.getName(), MetadataTracker.class)) + .map(registry -> registry.getTrackerForCore(core.getName(), ShardStatePublisher.class)) .orElseThrow(() -> new IllegalStateException("Cannot retrieve the Metadata tracker on this test core.")); // 1. First consistency check: ShardState must have the same values of CoreAdmin.SUMMARY report diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java index d7fc4b859..b7c2a9909 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java @@ -1,43 +1,43 @@ -/* - * #%L - * Alfresco Search Services - * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -package org.alfresco.solr; - -import java.util.concurrent.atomic.AtomicInteger; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * This class was moved from org.alfresco.solr.tracker.CoreTracker * The data in this class is relevant for a particular Solr index. */ public class TrackerState -{ - private static final Logger log = LoggerFactory.getLogger(TrackerState.class); +{ + private static final Logger log = LoggerFactory.getLogger(TrackerState.class); private volatile long lastChangeSetIdOnServer; @@ -61,22 +61,24 @@ public class TrackerState private volatile boolean running = false; + private boolean enabled; + private volatile boolean checkedFirstTransactionTime = false; private volatile boolean checkedFirstAclTransactionTime = false; private volatile boolean checkedLastAclTransactionTime = false; private volatile boolean checkedLastTransactionTime = false; - private volatile boolean check = false; + private volatile boolean check = false; // Handle Thread Safe operations - private volatile AtomicInteger trackerCycles = new AtomicInteger(0); - + private volatile AtomicInteger trackerCycles = new AtomicInteger(0); + private long timeToStopIndexing; private long lastGoodChangeSetCommitTimeInIndex; private long lastGoodTxCommitTimeInIndex; - private long timeBeforeWhichThereCanBeNoHoles; + private long timeBeforeWhichThereCanBeNoHoles; private volatile long lastStartTime = 0; public long getLastChangeSetIdOnServer() @@ -237,19 +239,19 @@ public class TrackerState public void setLastGoodTxCommitTimeInIndex(long lastGoodTxCommitTimeInIndex) { this.lastGoodTxCommitTimeInIndex = lastGoodTxCommitTimeInIndex; - } + } - public int getTrackerCycles() - { - return this.trackerCycles.get(); - } - - public synchronized void incrementTrackerCycles() - { - log.debug("incrementTrackerCycles from :" + trackerCycles); - this.trackerCycles.incrementAndGet(); - log.debug("incremented TrackerCycles to :" + trackerCycles); - } + public int getTrackerCycles() + { + return this.trackerCycles.get(); + } + + public synchronized void incrementTrackerCycles() + { + log.debug("incrementTrackerCycles from :" + trackerCycles); + this.trackerCycles.incrementAndGet(); + log.debug("incremented TrackerCycles to :" + trackerCycles); + } public long getTimeBeforeWhichThereCanBeNoHoles() { @@ -261,6 +263,14 @@ public class TrackerState this.timeBeforeWhichThereCanBeNoHoles = timeBeforeWhichThereCanBeNoHoles; } + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + /* * (non-Javadoc) * @see java.lang.Object#toString() @@ -284,10 +294,11 @@ public class TrackerState + ", checkedLastTransactionTime=" + this.checkedLastTransactionTime + ", checkedLastAclTransactionTime=" + this.checkedLastAclTransactionTime + ", check=" + check + + ", enabled=" + enabled + ", timeToStopIndexing=" + timeToStopIndexing + ", lastGoodChangeSetCommitTimeInIndex=" + lastGoodChangeSetCommitTimeInIndex + ", lastGoodTxCommitTimeInIndex=" + lastGoodTxCommitTimeInIndex - + ", timeBeforeWhichThereCanBeNoHoles=" + timeBeforeWhichThereCanBeNoHoles + + ", timeBeforeWhichThereCanBeNoHoles=" + timeBeforeWhichThereCanBeNoHoles + ",trackerCycles= " + trackerCycles + " ]"; } @@ -319,15 +330,15 @@ public class TrackerState public void setCheckedLastAclTransactionTime(boolean checkedLastAclTransactionTime) { this.checkedLastAclTransactionTime = checkedLastAclTransactionTime; - } + } - public long getLastStartTime() - { - return this.lastStartTime; - } - - public void setLastStartTime(long lastStartTime) - { - this.lastStartTime = lastStartTime; + public long getLastStartTime() + { + return this.lastStartTime; + } + + public void setLastStartTime(long lastStartTime) + { + this.lastStartTime = lastStartTime; } } diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 7d42ed0b7..8db3085d8 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -60,7 +60,7 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.5.0 + 1.6.0 unpack-solr-war