inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2025-09-10 14:11:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1476 from Alfresco/fix/MNT-22878_Search_Services_2.x_does_not_return_DB_TX_status_and_DB_TX_elements_in_nodeReport_of_SOLR_Admin_REST_API

Browse Source 
[MNT-22878] Amend NodeReport admin action on master/standalone instances
This commit is contained in:
Andrea Gazzarini
2022-08-09 19:13:19 +02:00
committed by GitHub
parent eabc597006 81957efe2f
commit ef6555da88
5 changed files with 167 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java
View File

@@ -32,8 +32,7 @@ import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.solr.adapters.IOpenBitSet;
import org.alfresco.solr.client.SOLRAPIClientFactory;
import org.alfresco.solr.config.ConfigUtil;
import org.alfresco.solr.io.interceptor.SharedSecretRequestInterceptor;
import org.alfresco.solr.security.SecretSharedPropertyCollector;
import org.alfresco.solr.tracker.AbstractTracker;
import org.alfresco.solr.tracker.AclTracker;
import org.alfresco.solr.tracker.ActivatableTracker;
import org.alfresco.solr.tracker.ShardStatePublisher;
@@ -48,7 +47,6 @@ import org.alfresco.solr.utils.Utils;
import org.alfresco.util.Pair;
import org.alfresco.util.shard.ExplicitShardingPolicy;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpRequestInterceptor;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
@@ -1027,12 +1025,12 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
coreNames().stream()
.filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName))
.filter(trackerRegistry::hasTrackersForCore)
.map(coreName -> new Pair<>(coreName, coreStatePublisher(coreName)))
.filter(coreNameAndPublisher -> coreNameAndPublisher.getSecond() != null)
.forEach(coreNameAndPublisher ->
.map(coreName -> new Pair<>(coreName, nodeStatusChecker(coreName)))
.filter(coreNameAndNodeChecker -> coreNameAndNodeChecker.getSecond() != null)
.forEach(coreNameAndNodeChecker ->
report.add(
coreNameAndPublisher.getFirst(),
buildNodeReport(coreNameAndPublisher.getSecond(), nodeid)));
coreNameAndNodeChecker.getFirst(),
buildNodeReport(coreNameAndNodeChecker.getSecond(), nodeid)));
return report;
}
@@ -2102,14 +2100,19 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
}
/**
* Returns, for the given core, the component which is in charge to publish the core state.
* Returns, for the given core, the tracker which is in charge to check the nodes status.
* Depending on the shard nature, master/standalone or slave, the tracker instance could be different.
* In addition, also the information that a given tracker returns about a given node, could differ (e.g.
* minimal in case of a slave node, detailed for master or standalone nodes).
*
* @param coreName the owning core name.
* @return the component which is in charge to publish the core state.
* @return the component which is in charge to check the nodes status.
*/
ShardStatePublisher coreStatePublisher(String coreName)
AbstractTracker nodeStatusChecker(String coreName)
{
return trackerRegistry.getTrackerForCore(coreName, ShardStatePublisher.class);
return isMasterOrStandalone(coreName)
? trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class)
: trackerRegistry.getTrackerForCore(coreName, ShardStatePublisher.class);
}
/**

30
search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java
View File

@@ -105,34 +105,24 @@ class HandlerReportHelper
}
}
static NamedList<Object> buildNodeReport(MetadataTracker tracker, Node node) throws JSONException
static NamedList<Object> buildNodeReport(AbstractTracker tracker, Node node) throws JSONException
{
NodeReport nodeReport = tracker.checkNode(node);
NamedList<Object> nr = new SimpleOrderedMap<>();
nr.add("Node DBID", nodeReport.getDbid());
nr.add("DB TX", nodeReport.getDbTx());
nr.add("DB TX status", nodeReport.getDbNodeStatus().toString());
if (nodeReport.getIndexLeafDoc() != null)
{
nr.add("Leaf tx in Index", nodeReport.getIndexLeafTx());
}
if (nodeReport.getIndexAuxDoc() != null)
{
nr.add("Aux tx in Index", nodeReport.getIndexAuxTx());
}
nr.add("Indexed Node Doc Count", nodeReport.getIndexedNodeDocCount());
return nr;
return buildNodeReport(tracker, node.getId());
}
static NamedList<Object> buildNodeReport(ShardStatePublisher publisher, Long dbid) throws JSONException
static NamedList<Object> buildNodeReport(AbstractTracker tracker, long dbid) throws JSONException
{
NodeReport nodeReport = publisher.checkNode(dbid);
NodeReport nodeReport = tracker.checkNode(dbid);
NamedList<Object> payload = new SimpleOrderedMap<>();
payload.add("Node DBID", nodeReport.getDbid());
if (publisher.isOnMasterOrStandalone())
boolean isOnMasterOrStandaloneMode =
tracker instanceof MetadataTracker
|| (tracker instanceof ShardStatePublisher
&& ((ShardStatePublisher)tracker).isOnMasterOrStandalone());
if (isOnMasterOrStandaloneMode)
{
ofNullable(nodeReport.getDbTx()).ifPresent(value -> payload.add("DB TX", value));
ofNullable(nodeReport.getDbNodeStatus()).map(Object::toString).ifPresent(value -> payload.add("DB TX Status", value));

117
search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerIT.java
View File

@@ -2,7 +2,7 @@
* #%L
* Alfresco Search Services
* %%
* Copyright (C) 2005 - 2020 Alfresco Software Limited
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
@@ -26,14 +26,37 @@
package org.alfresco.solr;
import org.alfresco.repo.search.adaptor.QueryConstants;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.SOLRAPIQueueClient;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.common.SolrException;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
import static java.util.Collections.singletonList;
import static java.util.Optional.ofNullable;
import static java.util.Optional.of;
import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
@SolrTestCaseJ4.SuppressSSL
public class AdminHandlerIT extends AbstractAlfrescoSolrIT
{
@@ -46,6 +69,21 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT
admin = getMultiCoreHandler();
}
@After
public void clearQueue()
{
SOLRAPIQueueClient.NODE_META_DATA_MAP.clear();
SOLRAPIQueueClient.TRANSACTION_QUEUE.clear();
SOLRAPIQueueClient.ACL_CHANGE_SET_QUEUE.clear();
SOLRAPIQueueClient.ACL_READERS_MAP.clear();
SOLRAPIQueueClient.ACL_MAP.clear();
SOLRAPIQueueClient.NODE_MAP.clear();
SOLRAPIQueueClient.NODE_CONTENT_MAP.clear();
clearIndex();
assertU(commit());
}
@Test(expected = SolrException.class)
public void testUnhandled() throws Exception
{
@@ -63,6 +101,73 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT
requestAction("removeCore");
}
@Test
public void nodeReportOnMasterOrStandaloneContainsTxInfo() throws Exception
{
var aclChangeSet = getAclChangeSet(1);
var acl = getAcl(aclChangeSet);
indexAclChangeSet(aclChangeSet,
singletonList(acl),
singletonList(getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null)));
var waitForQuery =
new BooleanQuery.Builder()
.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST))
.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(
QueryConstants.FIELD_S_ACLTXID, aclChangeSet.getId(),
aclChangeSet.getId() + 1,
true,
false), BooleanClause.Occur.MUST))
.build();
waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME);
var txn = getTransaction(0, 4);
var node = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
var nodeMetadata = getNodeMetaData(node, txn, acl, "mike", null, false);
indexTransaction(txn, singletonList(node), singletonList(nodeMetadata));
makeSureTransactionHasBeenIndexed(txn.getId());
var request = req(CoreAdminParams.ACTION, "nodereport", CoreAdminParams.NAME, getCore().getName());
var params =
new ModifiableSolrParams(request.getParams())
.set("nodeid", Long.toString(node.getId()));
request.setParams(params);
var response = new SolrQueryResponse();
admin.handleRequestBody(request, response);
var data = response.getValues();
var report =
ofNullable(data.get("report"))
.map(NamedList.class::cast)
.orElseThrow(() -> new AssertionError("'report' section not in response. Response was " + data));
var collection =
of(report.get("collection1"))
.map(NamedList.class::cast)
.orElseThrow(() -> new AssertionError("'collection' section not in response. Response was " + data));
of(collection.get("Node DBID"))
.map(Long.class::cast)
.filter(dbid -> node.getId() == dbid)
.orElseThrow(() -> new AssertionError("'Node DBID' mismatch or not found. Expected '" + node.getId() + "'Response was " + data));
of(collection.get("DB TX"))
.map(Long.class::cast)
.filter(dbtx -> txn.getId() == dbtx)
.orElseThrow(() -> new AssertionError("'DB TX' mismatch or not found. Expected '" + txn.getId() + "' Response was " + data));
of(collection.get("DB TX Status"))
.map(String.class::cast)
.filter("UPDATED"::equals)
.orElseThrow(() -> new AssertionError("'DB TX' mismatch or not found. Expected 'UPDATED', Response was " + data));
}
@Test
public void testhandledReports() throws Exception
{
@@ -77,4 +182,14 @@ public class AdminHandlerIT extends AbstractAlfrescoSolrIT
CoreAdminParams.NAME, getCore().getName()),
new SolrQueryResponse());
}
private void makeSureTransactionHasBeenIndexed(long transactionId) throws Exception
{
//Check for the TXN state stamp.
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!TX")), BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_TXID, transactionId, transactionId + 1, true, false), BooleanClause.Occur.MUST));
BooleanQuery waitForQuery = builder.build();
waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME);
}
}

4
search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerIT.java
View File

@@ -226,7 +226,7 @@ public class AlfrescoCoreAdminHandlerIT
when(trackerRegistry.getTrackerForCore(anyString(), eq(ShardStatePublisher.class)))
.thenReturn(coreStatePublisher);
assertSame(coreStatePublisher, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName"));
assertSame(coreStatePublisher, alfrescoCoreAdminHandler.nodeStatusChecker("ThisIsTheCoreName"));
}
@Test
@@ -236,7 +236,7 @@ public class AlfrescoCoreAdminHandlerIT
when(trackerRegistry.getTrackerForCore(anyString(), eq(ShardStatePublisher.class))).thenReturn(coreStateTracker);
assertSame(coreStateTracker, alfrescoCoreAdminHandler.coreStatePublisher("ThisIsTheCoreName"));
assertSame(coreStateTracker, alfrescoCoreAdminHandler.nodeStatusChecker("ThisIsTheCoreName"));
}
@Test

36
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java
View File

@@ -27,6 +27,7 @@
package org.alfresco.solr.client;
import static java.util.Optional.ofNullable;
import static java.util.stream.Collectors.toList;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -39,7 +40,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import org.alfresco.httpclient.Response;
import org.alfresco.repo.dictionary.NamespaceDAO;
@@ -96,7 +96,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
.peek(aclChangeSet -> {
maxTime.set(Math.max(aclChangeSet.getCommitTimeMs(), maxTime.get()));
maxId.set(Math.max(aclChangeSet.getId(), maxId.get()));})
.collect(Collectors.toList()), maxTime.get(), maxId.get());
.collect(toList()), maxTime.get(), maxId.get());
}
return new AclChangeSets(
@@ -106,7 +106,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
.peek(aclChangeSet -> {
maxTime.set(Math.max(aclChangeSet.getCommitTimeMs(), maxTime.get()));
maxId.set(Math.max(aclChangeSet.getId(), maxId.get()));})
.collect(Collectors.toList()), maxTime.get(), maxId.get());
.collect(toList()), maxTime.get(), maxId.get());
}
/**
@@ -129,7 +129,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
.map(AclChangeSet::getId)
.map(ACL_MAP::get)
.flatMap(Collection::stream)
.collect(Collectors.toList());
.collect(toList());
}
/**
@@ -148,7 +148,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
return acls.stream()
.map(Acl::getId)
.map(ACL_READERS_MAP::get)
.collect(Collectors.toList());
.collect(toList());
}
@@ -191,7 +191,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
.peek(txn -> {
maxTime.set(Math.max(txn.getCommitTimeMs(), maxTime.get()));
maxId.set(Math.max(txn.getId(), maxId.get()));})
.collect(Collectors.toList()), maxTime.get(), maxId.get());
.collect(toList()), maxTime.get(), maxId.get());
}
return new Transactions(
@@ -201,7 +201,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
.peek(txn -> {
maxTime.set(Math.max(txn.getCommitTimeMs(), maxTime.get()));
maxId.set(Math.max(txn.getId(), maxId.get()));})
.collect(Collectors.toList()), maxTime.get(), maxId.get());
.collect(toList()), maxTime.get(), maxId.get());
}
public List<Node> getNodes(GetNodesParameters parameters, int maxResults) throws IOException, JSONException
@@ -211,10 +211,22 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
throw new ConnectException("THROWING EXCEPTION, better be ready!");
}
return parameters.getTransactionIds().stream()
.map(NODE_MAP::get)
.flatMap(Collection::stream)
.collect(Collectors.toList());
return parameters.getTransactionIds() != null
? parameters.getTransactionIds().stream()
.map(NODE_MAP::get)
.flatMap(Collection::stream)
.collect(toList())
: NODE_MAP.values()
.stream()
.flatMap(Collection::stream)
.filter(node -> {
var fromNodeId = parameters.getFromNodeId();
var toNodeId = parameters.getToNodeId();
return (fromNodeId == null || node.getId() >= fromNodeId)
&&
(toNodeId == null || node.getId() <= toNodeId);})
.collect(toList());
}
@Override
@@ -230,7 +242,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
identifiers.stream()
.map(NODE_META_DATA_MAP::get)
.map(metadata -> getOnlyRequestedMetadata(metadata, params))
.collect(Collectors.toList()))
.collect(toList()))
.orElseGet(() ->
ofNullable(params.getFromNodeId())
.map(NODE_META_DATA_MAP::get)