diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 9fde7061f..372137ae6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -86,6 +86,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker private boolean txIntervalCommitTimeServiceAvailable = false; /** Whether the cascade tracking is enabled. */ private boolean cascadeTrackerEnabled = true; + + /** + * Transaction Id range to get the first transaction in database. + * 0-2000 by default. + */ + private Pair minTxnIdRange; public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -111,6 +117,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled(); + String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", "0-2000").split("-"); + minTxnIdRange = new Pair(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1])); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 @@ -243,7 +251,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker state.setCheckedFirstTransactionTime(true); log.info("No transactions found - no verification required"); - firstTransactions = client.getTransactions(null, 0L, null, 2000l, 1); + firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); if (!firstTransactions.getTransactions().isEmpty()) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); @@ -278,7 +286,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } } - firstTransactions = client.getTransactions(minCommitTime, 0L, null, 2000l, 1); + firstTransactions = client.getTransactions(minCommitTime, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); if (!firstTransactions.getTransactions().isEmpty()) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); @@ -311,7 +319,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { if (firstTransactions == null) { - firstTransactions = client.getTransactions(null, 0L, null, 2000l, 1); + firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); } setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index fab6b9e2e..189c181e0 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -182,6 +182,13 @@ solr.suggester.minSecsBetweenBuilds=3600 # solr.request.content.compress=false +# +# When checking repo and index consistency, first transaction is compared in both Repository and Index repositories. +# In order to get that initial transaction from database, 0-2000 range for txnId should be enough, but this parameter +# can be used when initial transaction Id is greater than 2000. +# +solr.initial.transaction.range=0-2000 + # # Limit the maximum text size of transformed content sent to the index - in bytes diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index d533ceb4d..1d33062d9 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -182,6 +182,13 @@ solr.suggester.minSecsBetweenBuilds=3600 # solr.request.content.compress=false +# +# When checking repo and index consistency, first transaction is compared in both Repository and Index repositories. +# In order to get that initial transaction from database, 0-2000 range for txnId should be enough, but this parameter +# can be used when initial transaction Id is greater than 2000. +# +solr.initial.transaction.range=0-2000 + # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerLargeTxnIdIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerLargeTxnIdIT.java new file mode 100644 index 000000000..da7a5ffa8 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerLargeTxnIdIT.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2005-2020 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import org.alfresco.repo.search.adaptor.lucene.QueryConstants; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.Transaction; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.LegacyNumericRangeQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.alfresco.solr.AlfrescoSolrUtils.ancestors; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; + +import java.util.Properties; + +/** + * Test environment having initial Transaction Id greater than default range 0-2000 + * @author aborroy + */ +@SolrTestCaseJ4.SuppressSSL +@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) +public class DistributedAlfrescoSolrTrackerLargeTxnIdIT extends AbstractAlfrescoDistributedIT +{ + + @BeforeClass + private static void initData() throws Throwable + { + Properties properties = new Properties(); + // Set a wider range for Initial Transaction Id + properties.put("solr.initial.transaction.range", "0-" + Long.MAX_VALUE); + initSolrServers(2, "DistributedAlfrescoSolrTrackerRaceTest", properties); + } + + @AfterClass + private static void destroyData() + { + dismissSolrServers(); + } + + @Test + public void testTracker() throws Exception + { + putHandleDefaults(); + + AclChangeSet aclChangeSet = getAclChangeSet(1); + + Acl acl = getAcl(aclChangeSet); + Acl acl2 = getAcl(aclChangeSet); + + AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + AclReaders aclReaders2 = getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null); + + // Transaction greater than 2000 is used, to check that "solr.initial.transaction.range" works as expected + Transaction txn = getTransaction(0, 2); + long txnCommitTimeMs = txn.getCommitTimeMs(); + + // Subtract from the commit time to go beyond hole retention + long backdatedCommitTimeMs = txnCommitTimeMs - 4600000; + txn.setCommitTimeMs(backdatedCommitTimeMs); + + //Next create two nodes to update for the transaction + Node folderNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + Node fileNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + Node errorNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + + // Next, create the node metadata for each node. + // Note: the error node metadata will cause an exception. + NodeMetaData folderMetaData = getNodeMetaData(folderNode, txn, acl, "mike", null, false); + NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", ancestors(folderMetaData.getNodeRef()), false); + NodeMetaData errorMetaData = getNodeMetaData(errorNode, txn, acl, "lisa", ancestors(folderMetaData.getNodeRef()), true); + + // Index the transaction, nodes, and nodeMetaDatas. + // Note that the content is automatically created by the test framework. + indexTransaction(txn, asList(errorNode, folderNode, fileNode), asList(errorMetaData, folderMetaData, fileMetaData)); + indexAclChangeSet(aclChangeSet, asList(acl, acl2), asList(aclReaders, aclReaders2)); + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST)); + builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST)); + BooleanQuery waitForQuery = builder.build(); + waitForDocCountAllCores(waitForQuery, 1, 80000); + + } +} \ No newline at end of file