From 41882b3bae6ad59414f0ec2156f97a34b41de14f Mon Sep 17 00:00:00 2001 From: Elia Date: Tue, 23 Aug 2022 17:58:05 +0200 Subject: [PATCH] [MNT-23072] fix index last transaction error in dbid_range sharding added integration test (cherry picked from commit 75067e492161acafc1d0b1f90e494fd38a96a066) --- .../solr/tracker/MetadataTracker.java | 12 ++-- ...ributedDbidRangeAlfrescoSolrTrackerIT.java | 67 ++++++++++++++++--- .../solr/client/SOLRAPIQueueClient.java | 20 ++++++ 3 files changed, 82 insertions(+), 17 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index be14895b0..085b356ca 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -94,10 +94,6 @@ public class MetadataTracker extends ActivatableTracker private final ConcurrentLinkedQueue nodesToIndex = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue nodesToPurge = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue<>(); - - private final boolean isRunningInProduction = - !Boolean.parseBoolean(System.getProperty("alfresco.test", "false")); - private ForkJoinPool forkJoinPool; // Share run and write locks across all MetadataTracker threads @@ -186,7 +182,7 @@ public class MetadataTracker extends ActivatableTracker // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 - if (checkRepoServicesAvailability && isRunningInProduction) + if (checkRepoServicesAvailability) { // Try invoking getNextTxCommitTime service try @@ -863,7 +859,7 @@ public class MetadataTracker extends ActivatableTracker latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); latestTransaction.setId(transactions.getMaxTxnId()); - if (!isTransactionIndexed(latestTransaction)) + if (isTransactionToBeIndexed(latestTransaction)) { transactions = new Transactions(Collections.singletonList(latestTransaction), transactions.getMaxTxnCommitTime(), transactions.getMaxTxnId()); @@ -880,7 +876,7 @@ public class MetadataTracker extends ActivatableTracker return transactions; } - private boolean isTransactionIndexed(Transaction transaction) + private boolean isTransactionToBeIndexed(Transaction transaction) { try { @@ -990,7 +986,7 @@ public class MetadataTracker extends ActivatableTracker final AtomicInteger counterTransaction = new AtomicInteger(); Collection> txBatches = transactions.getTransactions().stream() .peek(txnsFound::add) - .filter(this::isTransactionIndexed) + .filter(this::isTransactionToBeIndexed) .collect(Collectors.groupingBy(transaction -> counterTransaction.getAndAdd( (int) (transaction.getDeletes() + transaction.getUpdates())) / transactionDocsBatchSize)) .values(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java index fc5d52c90..d825370b0 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java @@ -37,6 +37,7 @@ import org.alfresco.solr.client.Transaction; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Properties; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_DOC_TYPE; +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; @@ -65,7 +67,7 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD @BeforeClass public static void initData() throws Throwable { - initSolrServers(2, getSimpleClassName(), getShardMethod()); + initSolrServers(3, getSimpleClassName(), getShardMethod()); } @AfterClass @@ -73,13 +75,14 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD { dismissSolrServers(); } - - @Test - public void testDbIdRange() throws Exception - { - putHandleDefaults(); - int numAcls = 250; + @After + public void deleteDataFromIndex() throws Exception { + deleteByQueryAllClients("*:*"); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 0, MAX_WAIT_TIME); + } + + private List createAcls(int numAcls){ AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls); List bulkAcls = new ArrayList<>(); @@ -99,6 +102,17 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD bulkAcls, bulkAclReaders); + return bulkAcls; + } + + @Test + public void testDbIdRange() throws Exception + { + putHandleDefaults(); + + int numAcls = 250; + var bulkAcls = createAcls(numAcls); + int numNodes = 150; List nodes = new ArrayList<>(); List nodeMetaDatas = new ArrayList<>(); @@ -108,14 +122,14 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD for(int i=0; i getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId) + { + List transactions = TRANSACTION_QUEUE.stream() + .filter(txn -> NODE_MAP.get(txn.getId()) + .stream() + .anyMatch(node -> node.getId() >= fromNodeId && node.getId() <= toNodeId)) + .collect(Collectors.toList()); + + if (transactions.size() > 0){ + return new Pair<>(transactions.get(0).getCommitTimeMs(), transactions.get(transactions.size() - 1).getCommitTimeMs()); + } else { + return new Pair<>(-1l, -1l); + } + } public Transactions getTransactions(Long fromCommitTime, Long minTxnId, Long toCommitTime, Long maxTxnId, int maxResults) throws IOException, JSONException {