diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index cdd8211ba..bab9242bf 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -32,8 +32,7 @@ import org.alfresco.service.cmr.repository.StoreRef; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.SOLRAPIClientFactory; import org.alfresco.solr.config.ConfigUtil; -import org.alfresco.solr.io.interceptor.SharedSecretRequestInterceptor; -import org.alfresco.solr.security.SecretSharedPropertyCollector; +import org.alfresco.solr.tracker.AbstractTracker; import org.alfresco.solr.tracker.AclTracker; import org.alfresco.solr.tracker.ActivatableTracker; import org.alfresco.solr.tracker.ShardStatePublisher; @@ -48,7 +47,6 @@ import org.alfresco.solr.utils.Utils; import org.alfresco.util.Pair; import org.alfresco.util.shard.ExplicitShardingPolicy; import org.apache.commons.io.FileUtils; -import org.apache.http.HttpRequestInterceptor; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.SolrParams; @@ -1027,12 +1025,12 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler coreNames().stream() .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(trackerRegistry::hasTrackersForCore) - .map(coreName -> new Pair<>(coreName, coreStatePublisher(coreName))) - .filter(coreNameAndPublisher -> coreNameAndPublisher.getSecond() != null) - .forEach(coreNameAndPublisher -> + .map(coreName -> new Pair<>(coreName, nodeStatusChecker(coreName))) + .filter(coreNameAndNodeChecker -> coreNameAndNodeChecker.getSecond() != null) + .forEach(coreNameAndNodeChecker -> report.add( - coreNameAndPublisher.getFirst(), - buildNodeReport(coreNameAndPublisher.getSecond(), nodeid))); + coreNameAndNodeChecker.getFirst(), + buildNodeReport(coreNameAndNodeChecker.getSecond(), nodeid))); return report; } @@ -2102,14 +2100,19 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } /** - * Returns, for the given core, the component which is in charge to publish the core state. + * Returns, for the given core, the tracker which is in charge to check the nodes status. + * Depending on the shard nature, master/standalone or slave, the tracker instance could be different. + * In addition, also the information that a given tracker returns about a given node, could differ (e.g. + * minimal in case of a slave node, detailed for master or standalone nodes). * * @param coreName the owning core name. - * @return the component which is in charge to publish the core state. + * @return the component which is in charge to check the nodes status. */ - ShardStatePublisher coreStatePublisher(String coreName) + AbstractTracker nodeStatusChecker(String coreName) { - return trackerRegistry.getTrackerForCore(coreName, ShardStatePublisher.class); + return isMasterOrStandalone(coreName) + ? trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class) + : trackerRegistry.getTrackerForCore(coreName, ShardStatePublisher.class); } /** diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java index 3773fbcb4..71aa94c87 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java @@ -105,34 +105,24 @@ class HandlerReportHelper } } - static NamedList buildNodeReport(MetadataTracker tracker, Node node) throws JSONException + static NamedList buildNodeReport(AbstractTracker tracker, Node node) throws JSONException { - NodeReport nodeReport = tracker.checkNode(node); - - NamedList nr = new SimpleOrderedMap<>(); - nr.add("Node DBID", nodeReport.getDbid()); - nr.add("DB TX", nodeReport.getDbTx()); - nr.add("DB TX status", nodeReport.getDbNodeStatus().toString()); - if (nodeReport.getIndexLeafDoc() != null) - { - nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx()); - } - if (nodeReport.getIndexAuxDoc() != null) - { - nr.add("Aux tx in Index", nodeReport.getIndexAuxTx()); - } - nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount()); - return nr; + return buildNodeReport(tracker, node.getId()); } - static NamedList buildNodeReport(ShardStatePublisher publisher, Long dbid) throws JSONException + static NamedList buildNodeReport(AbstractTracker tracker, long dbid) throws JSONException { - NodeReport nodeReport = publisher.checkNode(dbid); + NodeReport nodeReport = tracker.checkNode(dbid); NamedList payload = new SimpleOrderedMap<>(); payload.add("Node DBID", nodeReport.getDbid()); - if (publisher.isOnMasterOrStandalone()) + boolean isOnMasterOrStandaloneMode = + tracker instanceof MetadataTracker + || (tracker instanceof ShardStatePublisher + && ((ShardStatePublisher)tracker).isOnMasterOrStandalone()); + + if (isOnMasterOrStandaloneMode) { ofNullable(nodeReport.getDbTx()).ifPresent(value -> payload.add("DB TX", value)); ofNullable(nodeReport.getDbNodeStatus()).map(Object::toString).ifPresent(value -> payload.add("DB TX Status", value)); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java index 56887c032..94b0b1dad 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Search Services * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -26,14 +26,37 @@ package org.alfresco.solr; +import org.alfresco.repo.search.adaptor.QueryConstants; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.SOLRAPIQueueClient; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.LegacyNumericRangeQuery; +import org.apache.lucene.search.TermQuery; import org.apache.solr.common.SolrException; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.response.SolrQueryResponse; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import static java.util.Collections.singletonList; +import static java.util.Optional.ofNullable; +import static java.util.Optional.of; +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; + @SolrTestCaseJ4.SuppressSSL public class AdminHandlerIT extends AbstractAlfrescoSolrIT { @@ -46,6 +69,21 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT admin = getMultiCoreHandler(); } + @After + public void clearQueue() + { + SOLRAPIQueueClient.NODE_META_DATA_MAP.clear(); + SOLRAPIQueueClient.TRANSACTION_QUEUE.clear(); + SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear(); + SOLRAPIQueueClient.ACL_READERS_MAP.clear(); + SOLRAPIQueueClient.ACL_MAP.clear(); + SOLRAPIQueueClient.NODE_MAP.clear(); + SOLRAPIQueueClient.NODE_CONTENT_MAP.clear(); + + clearIndex(); + assertU(commit()); + } + @Test(expected = SolrException.class) public void testUnhandled() throws Exception { @@ -63,6 +101,73 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT requestAction("removeCore"); } + @Test + public void nodeReportOnMasterOrStandaloneContainsTxInfo() throws Exception + { + var aclChangeSet = getAclChangeSet(1); + var acl = getAcl(aclChangeSet); + + indexAclChangeSet(aclChangeSet, + singletonList(acl), + singletonList(getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null))); + + var waitForQuery = + new BooleanQuery.Builder() + .add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST)) + .add(new BooleanClause(LegacyNumericRangeQuery.newLongRange( + QueryConstants.FIELD_S_ACLTXID, aclChangeSet.getId(), + aclChangeSet.getId() + 1, + true, + false), BooleanClause.Occur.MUST)) + .build(); + waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME); + + var txn = getTransaction(0, 4); + var node = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + var nodeMetadata = getNodeMetaData(node, txn, acl, "mike", null, false); + + indexTransaction(txn, singletonList(node), singletonList(nodeMetadata)); + + makeSureTransactionHasBeenIndexed(txn.getId()); + + var request = req(CoreAdminParams.ACTION, "nodereport", CoreAdminParams.NAME, getCore().getName()); + var params = + new ModifiableSolrParams(request.getParams()) + .set("nodeid", Long.toString(node.getId())); + request.setParams(params); + var response = new SolrQueryResponse(); + + admin.handleRequestBody(request, response); + + var data = response.getValues(); + + var report = + ofNullable(data.get("report")) + .map(NamedList.class::cast) + .orElseThrow(() -> new AssertionError("'report' section not in response. Response was " + data)); + + var collection = + of(report.get("collection1")) + .map(NamedList.class::cast) + .orElseThrow(() -> new AssertionError("'collection' section not in response. Response was " + data)); + + of(collection.get("Node DBID")) + .map(Long.class::cast) + .filter(dbid -> node.getId() == dbid) + .orElseThrow(() -> new AssertionError("'Node DBID' mismatch or not found. Expected '" + node.getId() + "'Response was " + data)); + + of(collection.get("DB TX")) + .map(Long.class::cast) + .filter(dbtx -> txn.getId() == dbtx) + .orElseThrow(() -> new AssertionError("'DB TX' mismatch or not found. Expected '" + txn.getId() + "' Response was " + data)); + + of(collection.get("DB TX Status")) + .map(String.class::cast) + .filter("UPDATED"::equals) + .orElseThrow(() -> new AssertionError("'DB TX' mismatch or not found. Expected 'UPDATED', Response was " + data)); + + } + @Test public void testhandledReports() throws Exception { @@ -77,4 +182,14 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT CoreAdminParams.NAME, getCore().getName()), new SolrQueryResponse()); } + + private void makeSureTransactionHasBeenIndexed(long transactionId) throws Exception + { + //Check for the TXN state stamp. + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!TX")), BooleanClause.Occur.MUST)); + builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_TXID, transactionId, transactionId + 1, true, false), BooleanClause.Occur.MUST)); + BooleanQuery waitForQuery = builder.build(); + waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME); + } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java index d380eeeb2..ed14dfbd3 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java @@ -226,7 +226,7 @@ public class AlfrescoCoreAdminHandlerIT when(trackerRegistry.getTrackerForCore(anyString(), eq(ShardStatePublisher.class))) .thenReturn(coreStatePublisher); - assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); + assertSame(coreStatePublisher, alfrescoCoreAdminHandler.nodeStatusChecker("ThisIsTheCoreName")); } @Test @@ -236,7 +236,7 @@ public class AlfrescoCoreAdminHandlerIT when(trackerRegistry.getTrackerForCore(anyString(), eq(ShardStatePublisher.class))).thenReturn(coreStateTracker); - assertSame(coreStateTracker, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName")); + assertSame(coreStateTracker, alfrescoCoreAdminHandler.nodeStatusChecker("ThisIsTheCoreName")); } @Test diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index e3f69b52d..04d45173a 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -27,6 +27,7 @@ package org.alfresco.solr.client; import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toList; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -39,7 +40,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; import org.alfresco.httpclient.Response; import org.alfresco.repo.dictionary.NamespaceDAO; @@ -96,7 +96,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .peek(aclChangeSet -> { maxTime.set(Math.max(aclChangeSet.getCommitTimeMs(), maxTime.get())); maxId.set(Math.max(aclChangeSet.getId(), maxId.get()));}) - .collect(Collectors.toList()), maxTime.get(), maxId.get()); + .collect(toList()), maxTime.get(), maxId.get()); } return new AclChangeSets( @@ -106,7 +106,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .peek(aclChangeSet -> { maxTime.set(Math.max(aclChangeSet.getCommitTimeMs(), maxTime.get())); maxId.set(Math.max(aclChangeSet.getId(), maxId.get()));}) - .collect(Collectors.toList()), maxTime.get(), maxId.get()); + .collect(toList()), maxTime.get(), maxId.get()); } /** @@ -129,7 +129,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .map(AclChangeSet::getId) .map(ACL_MAP::get) .flatMap(Collection::stream) - .collect(Collectors.toList()); + .collect(toList()); } /** @@ -148,7 +148,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient return acls.stream() .map(Acl::getId) .map(ACL_READERS_MAP::get) - .collect(Collectors.toList()); + .collect(toList()); } @@ -191,7 +191,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .peek(txn -> { maxTime.set(Math.max(txn.getCommitTimeMs(), maxTime.get())); maxId.set(Math.max(txn.getId(), maxId.get()));}) - .collect(Collectors.toList()), maxTime.get(), maxId.get()); + .collect(toList()), maxTime.get(), maxId.get()); } return new Transactions( @@ -201,7 +201,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .peek(txn -> { maxTime.set(Math.max(txn.getCommitTimeMs(), maxTime.get())); maxId.set(Math.max(txn.getId(), maxId.get()));}) - .collect(Collectors.toList()), maxTime.get(), maxId.get()); + .collect(toList()), maxTime.get(), maxId.get()); } public List getNodes(GetNodesParameters parameters, int maxResults) throws IOException, JSONException @@ -211,10 +211,22 @@ public class SOLRAPIQueueClient extends SOLRAPIClient throw new ConnectException("THROWING EXCEPTION, better be ready!"); } - return parameters.getTransactionIds().stream() - .map(NODE_MAP::get) - .flatMap(Collection::stream) - .collect(Collectors.toList()); + return parameters.getTransactionIds() != null + ? parameters.getTransactionIds().stream() + .map(NODE_MAP::get) + .flatMap(Collection::stream) + .collect(toList()) + : NODE_MAP.values() + .stream() + .flatMap(Collection::stream) + .filter(node -> { + var fromNodeId = parameters.getFromNodeId(); + var toNodeId = parameters.getToNodeId(); + + return (fromNodeId == null || node.getId() >= fromNodeId) + && + (toNodeId == null || node.getId() <= toNodeId);}) + .collect(toList()); } @Override @@ -230,7 +242,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient identifiers.stream() .map(NODE_META_DATA_MAP::get) .map(metadata -> getOnlyRequestedMetadata(metadata, params)) - .collect(Collectors.toList())) + .collect(toList())) .orElseGet(() -> ofNullable(params.getFromNodeId()) .map(NODE_META_DATA_MAP::get)