diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 556e58a03..ebfa390a9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -94,10 +94,6 @@ public class MetadataTracker extends ActivatableTracker private final ConcurrentLinkedQueue nodesToIndex = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue nodesToPurge = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue<>(); - - private final boolean isRunningInProduction = - !Boolean.parseBoolean(System.getProperty("alfresco.test", "false")); - private ForkJoinPool forkJoinPool; // Share run and write locks across all MetadataTracker threads @@ -193,7 +189,7 @@ public class MetadataTracker extends ActivatableTracker // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 - if (checkRepoServicesAvailability && isRunningInProduction) + if (checkRepoServicesAvailability) { // Try invoking getNextTxCommitTime service try @@ -870,7 +866,7 @@ public class MetadataTracker extends ActivatableTracker latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); latestTransaction.setId(transactions.getMaxTxnId()); - if (!isTransactionIndexed(latestTransaction)) + if (isTransactionToBeIndexed(latestTransaction)) { transactions = new Transactions(Collections.singletonList(latestTransaction), transactions.getMaxTxnCommitTime(), transactions.getMaxTxnId()); @@ -887,7 +883,7 @@ public class MetadataTracker extends ActivatableTracker return transactions; } - private boolean isTransactionIndexed(Transaction transaction) + boolean isTransactionToBeIndexed(Transaction transaction) { try { @@ -997,7 +993,7 @@ public class MetadataTracker extends ActivatableTracker final AtomicInteger counterTransaction = new AtomicInteger(); Collection> txBatches = transactions.getTransactions().stream() .peek(txnsFound::add) - .filter(this::isTransactionIndexed) + .filter(this::isTransactionToBeIndexed) .collect(Collectors.groupingBy(transaction -> counterTransaction.getAndAdd( (int) (transaction.getDeletes() + transaction.getUpdates())) / transactionDocsBatchSize)) .values(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java index fc5d52c90..d734399e1 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Search Services * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -23,7 +23,6 @@ * along with Alfresco. If not, see . * #L% */ - package org.alfresco.solr.tracker; import org.alfresco.solr.AbstractAlfrescoDistributedIT; @@ -33,10 +32,12 @@ import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclReaders; import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.SOLRAPIQueueClient; import org.alfresco.solr.client.Transaction; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Properties; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_DOC_TYPE; +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; @@ -54,7 +56,6 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getNode; import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; -import static org.alfresco.solr.AlfrescoSolrUtils.list; /** * @author Joel @@ -65,7 +66,7 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD @BeforeClass public static void initData() throws Throwable { - initSolrServers(2, getSimpleClassName(), getShardMethod()); + initSolrServers(3, getSimpleClassName(), getShardMethod()); } @AfterClass @@ -73,19 +74,25 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD { dismissSolrServers(); } - - @Test - public void testDbIdRange() throws Exception - { - putHandleDefaults(); - int numAcls = 250; + @After + public void deleteDataFromIndex() throws Exception + { + SOLRAPIQueueClient.TRANSACTION_QUEUE.clear(); + SOLRAPIQueueClient.NODE_MAP.clear(); + deleteByQueryAllClients("*:*"); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 0, MAX_WAIT_TIME); + } + + private List createAcls(int numAcls) + { AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls); List bulkAcls = new ArrayList<>(); List bulkAclReaders = new ArrayList<>(); - for(int i=0; i nodes = new ArrayList<>(); List nodeMetaDatas = new ArrayList<>(); @@ -108,14 +126,14 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD for(int i=0; i nodes = getNodes(); - when(repositoryClient.getNodes(any(GetNodesParameters.class), anyInt())).thenReturn(nodes); - - List nodes4Tx = this.metadataTracker.getFullNodesForDbTransaction(TX_ID); - - assertSame(nodes4Tx, nodes); - } } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index 04d45173a..ded8cd45f 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -36,15 +36,18 @@ import java.net.ConnectException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.alfresco.httpclient.Response; import org.alfresco.repo.dictionary.NamespaceDAO; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.service.namespace.QName; +import org.alfresco.util.Pair; import org.apache.http.HttpStatus; import org.json.JSONException; @@ -161,6 +164,30 @@ public class SOLRAPIQueueClient extends SOLRAPIClient return Collections.emptyList(); } + public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws NoSuchMethodException + { + throw new NoSuchMethodException(); + } + + public Pair getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId) + { + List transactionCommitTimestamps = TRANSACTION_QUEUE.stream() + .filter(txn -> NODE_MAP.get(txn.getId()) + .stream() + .anyMatch(node -> node.getId() >= fromNodeId && node.getId() <= toNodeId)) + .map(tx -> tx.getCommitTimeMs()) + .sorted() + .collect(Collectors.toList()); + + if (transactionCommitTimestamps.size() > 0) + { + return new Pair<>( transactionCommitTimestamps.get(0), transactionCommitTimestamps.get(transactionCommitTimestamps.size() - 1)); + } + else + { + return new Pair<>(-1l, -1l); + } + } public Transactions getTransactions(Long fromCommitTime, Long minTxnId, Long toCommitTime, Long maxTxnId, int maxResults) throws IOException, JSONException {