inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1105 from Alfresco/fix/MNT-22094_StoreMaxChangeSet

Browse Source 
MNT-22094 Store max change set.

(cherry picked from commit bbac8534b89bc4fcb925126ae836a90e9b112c09)
This commit is contained in:
Tom Page
2021-02-26 14:08:38 +00:00
committed by Tom Page
parent 1d9e497352
commit 00d06831f6
3 changed files with 224 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
+44 -43
View File
@@ -46,7 +46,6 @@ import java.util.stream.Collectors;
import com.google.common.collect.Lists;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.alfresco.solr.AclReport;
import org.alfresco.solr.BoundedDeque;
import org.alfresco.solr.InformationServer;
@@ -77,7 +76,7 @@ public class AclTracker extends ActivatableTracker
private static final int DEFAULT_ACL_TRACKER_MAX_PARALLELISM = 32;
private static final long DEFAULT_ACL_TRACKER_TIMESTEP = TIME_STEP_1_HR_IN_MS;
private static final long INITIAL_MAX_ACL_CHANGE_SET_ID = 2000L;
protected static final long INITIAL_MAX_ACL_CHANGE_SET_ID = 2000L;
private static final int MAX_NUMBER_OF_ACL_CHANGE_SETS = 2000;
private static final long MAX_TIME_STEP = TIME_STEP_32_DAYS_IN_MS;
@@ -405,17 +404,24 @@ public class AclTracker extends ActivatableTracker
/**
* Checks the first and last TX time
*/
private void checkRepoAndIndexConsistency(TrackerState state) throws AuthenticationException, IOException, JSONException
protected void checkRepoAndIndexConsistency(TrackerState state) throws AuthenticationException, IOException, JSONException
{
AclChangeSets firstChangeSets = null;
if (state.getLastGoodChangeSetCommitTimeInIndex() != 0 && state.isCheckedFirstAclTransactionTime() && state.isCheckedLastAclTransactionTime())
{
// Verification done previously.
return;
}
// Load the first ACL change sets from the Repository.
AclChangeSets firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
if (state.getLastGoodChangeSetCommitTimeInIndex() == 0)
{
state.setCheckedLastAclTransactionTime(true);
state.setCheckedFirstAclTransactionTime(true);
LOGGER.info("[CORE {}] - No acl transactions found - no verification required", coreName);
firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
{
AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0);
@@ -425,54 +431,49 @@ public class AclTracker extends ActivatableTracker
}
}
if (!state.isCheckedFirstAclTransactionTime())
if (!state.isCheckedFirstAclTransactionTime() && !firstChangeSets.getAclChangeSets().isEmpty())
{
firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
AclChangeSet firstAclChangeSet = firstChangeSets.getAclChangeSets().get(0);
long firstAclTxId = firstAclChangeSet.getId();
long firstAclTxCommitTime = firstAclChangeSet.getCommitTimeMs();
int setSize = this.infoSrv.getAclTxDocsSize(Long.toString(firstAclTxId),
Long.toString(firstAclTxCommitTime));
if (setSize == 0)
{
AclChangeSet firstAclChangeSet= firstChangeSets.getAclChangeSets().get(0);
long firstAclTxId = firstAclChangeSet.getId();
long firstAclTxCommitTime = firstAclChangeSet.getCommitTimeMs();
int setSize = this.infoSrv.getAclTxDocsSize(Long.toString(firstAclTxId),
Long.toString(firstAclTxCommitTime));
if (setSize == 0)
{
LOGGER.error("[CORE {}] First acl transaction was not found with the correct timestamp.", coreName);
LOGGER.error("SOLR has successfully connected to your repository " +
"however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes " +
"also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial acl transaction not found with correct timestamp");
}
else if (setSize == 1)
{
state.setCheckedFirstTransactionTime(true);
LOGGER.info("[CORE {}] Verified first acl transaction and timestamp in index", coreName);
}
else
{
LOGGER.warn("[CORE {}] Duplicate initial acl transaction found with correct timestamp", coreName);
}
LOGGER.error("[CORE {}] First acl transaction was not found with the correct timestamp.", coreName);
LOGGER.error("SOLR has successfully connected to your repository " +
"however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes " +
"also need to be re-built to match the database.");
LOGGER.error("Notice that SOLR will continue to track the repository, but the index may be corrupted.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial ACL transaction from DB with Id=" + firstAclTxId
+ " and Timestamp=" + firstAclTxCommitTime + " was not found in Solr core.");
}
else if (setSize == 1)
{
state.setCheckedFirstAclTransactionTime(true);
LOGGER.info("[CORE {}] Verified first acl transaction and timestamp in index", coreName);
}
else
{
LOGGER.warn("[CORE {}] Duplicate initial acl transaction found with correct timestamp", coreName);
}
}
// Checks that the last aclTxId in solr is <= last aclTxId in repo
if (!state.isCheckedLastAclTransactionTime())
{
if (firstChangeSets == null)
{
firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
}
setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state);
Long maxChangeSetCommitTimeInRepo = firstChangeSets.getMaxChangeSetCommitTime();
Long maxChangeSetIdInRepo = firstChangeSets.getMaxChangeSetId();
if (maxChangeSetCommitTimeInRepo != null && maxChangeSetIdInRepo != null)
{
// We know the server has at least as many transactions as the index.
state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTimeInRepo);
state.setLastChangeSetIdOnServer(maxChangeSetIdInRepo);
AclChangeSet maxAclTxInIndex = this.infoSrv.getMaxAclChangeSetIdAndCommitTimeInIndex();
if (maxAclTxInIndex.getCommitTimeMs() > maxChangeSetCommitTimeInRepo)
{
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclTrackerTest.java
+179
View File
@@ -0,0 +1,179 @@
/*
* #%L
* Alfresco Search Services
* %%
* Copyright (C) 2005 - 2020 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.tracker;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static org.alfresco.solr.tracker.AclTracker.INITIAL_MAX_ACL_CHANGE_SET_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.TrackerState;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclChangeSets;
import org.alfresco.solr.client.SOLRAPIClient;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
/** Unit tests for the {@link AclTracker}. */
public class AclTrackerTest
{
/** The class under test. */
@InjectMocks
private AclTracker aclTracker;
/** The class that gets information from the Repository. */
@Mock
private SOLRAPIClient repositoryClient;
/** The class that gets information from Solr. */
@Mock
private InformationServer solrInformationServer;
@Before
public void setUp()
{
aclTracker = new AclTracker();
openMocks(this);
}
/** Check that during the first run (with an empty repository and index) then verification is successful. */
@Test
public void testCheckRepoAndIndexConsistency_firstRun_success() throws Exception
{
TrackerState trackerState = new TrackerState();
AclChangeSets firstChangeSets = new AclChangeSets(emptyList());
when(repositoryClient.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1)).thenReturn(firstChangeSets);
// Call the method under test.
aclTracker.checkRepoAndIndexConsistency(trackerState);
assertTrue("Expected first ACL transaction to have been checked.", trackerState.isCheckedFirstAclTransactionTime());
assertTrue("Expected last ACL transaction to have been checked.", trackerState.isCheckedLastAclTransactionTime());
}
/** Check that subsequent checks of a running index don't make expensive requests. */
@Test
public void testCheckRepoAndIndexConsistency_alreadyInitialised_success() throws Exception
{
TrackerState trackerState = new TrackerState();
trackerState.setLastGoodChangeSetCommitTimeInIndex(8000L);
trackerState.setCheckedFirstAclTransactionTime(true);
trackerState.setCheckedLastAclTransactionTime(true);
// Call the method under test.
aclTracker.checkRepoAndIndexConsistency(trackerState);
// Check that we don't make any expensive calls to the index or the repo.
verifyNoInteractions(repositoryClient, solrInformationServer);
}
/** Check that during the first run (with data in the repo but an empty index) then verification is successful. */
@Test
public void testCheckRepoAndIndexConsistency_populatedRepoEmptyIndex_success() throws Exception
{
TrackerState trackerState = new TrackerState();
AclChangeSets firstChangeSets = new AclChangeSets(asList(new AclChangeSet(1, 1000, 2)), 8000L, 8L);
when(repositoryClient.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1)).thenReturn(firstChangeSets);
// Call the method under test.
aclTracker.checkRepoAndIndexConsistency(trackerState);
assertTrue("Expected first ACL transaction to have been checked.", trackerState.isCheckedFirstAclTransactionTime());
assertTrue("Expected last ACL transaction to have been checked.", trackerState.isCheckedLastAclTransactionTime());
assertEquals("Expected last good change set commit time to be loaded from repository.",
trackerState.getLastGoodChangeSetCommitTimeInIndex(), 1000L);
assertEquals("Expected last change set commit time to be loaded from repository.",
trackerState.getLastChangeSetCommitTimeOnServer(), 1000L);
assertEquals("Expected last change set id to be loaded from repository.",
trackerState.getLastChangeSetIdOnServer(), 1);
}
/** Check that after downtime the validation is successful. */
@Test
public void testCheckRepoAndIndexConsistency_afterRestart_success() throws Exception
{
TrackerState trackerState = new TrackerState();
trackerState.setLastGoodChangeSetCommitTimeInIndex(8000L);
AclChangeSets firstChangeSets = new AclChangeSets(asList(new AclChangeSet(1, 1000, 2)), 8000L, 8L);
when(repositoryClient.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1)).thenReturn(firstChangeSets);
when(solrInformationServer.getAclTxDocsSize("1", "1000")).thenReturn(1);
// The index is behind the repo.
AclChangeSet lastIndexedChangeSet = new AclChangeSet(7, 7000, 7);
when(solrInformationServer.getMaxAclChangeSetIdAndCommitTimeInIndex()).thenReturn(lastIndexedChangeSet);
// Call the method under test.
aclTracker.checkRepoAndIndexConsistency(trackerState);
assertTrue("Expected first ACL transaction to have been checked.", trackerState.isCheckedFirstAclTransactionTime());
assertTrue("Expected last ACL transaction to have been checked.", trackerState.isCheckedLastAclTransactionTime());
}
/** Check that if the index is populated but missing the first ACL transaction then we get an exception. */
@Test(expected = AlfrescoRuntimeException.class)
public void testCheckRepoAndIndexConsistency_indexMissingFirstACLTx_runtimeException() throws Exception
{
TrackerState trackerState = new TrackerState();
trackerState.setLastGoodChangeSetCommitTimeInIndex(8000L);
AclChangeSets firstChangeSets = new AclChangeSets(asList(new AclChangeSet(1, 1000, 2)), 8000L, 8L);
when(repositoryClient.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1)).thenReturn(firstChangeSets);
// The first ACL transaction was not found in Solr.
when(solrInformationServer.getAclTxDocsSize("1", "1000")).thenReturn(0);
// Call the method under test.
aclTracker.checkRepoAndIndexConsistency(trackerState);
}
/** Check that if the last ACL in the index is after the last ACL in the repository then we get an exception. */
@Test (expected = AlfrescoRuntimeException.class)
public void testCheckRepoAndIndexConsistency_indexAheadOfRepo_runtimeException() throws Exception
{
TrackerState trackerState = new TrackerState();
trackerState.setLastGoodChangeSetCommitTimeInIndex(8000L);
AclChangeSets firstChangeSets = new AclChangeSets(asList(new AclChangeSet(1, 1000, 2)), 8000L, 8L);
when(repositoryClient.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1)).thenReturn(firstChangeSets);
when(solrInformationServer.getAclTxDocsSize("1", "1000")).thenReturn(1);
// The index contains an ACL after the last one from the server (id 8 at time 8000L).
AclChangeSet lastIndexedChangeSet = new AclChangeSet(9, 9000, 9);
when(solrInformationServer.getMaxAclChangeSetIdAndCommitTimeInIndex()).thenReturn(lastIndexedChangeSet);
// Call the method under test.
aclTracker.checkRepoAndIndexConsistency(trackerState);
}
}
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java
+1 -1
View File
@@ -43,7 +43,7 @@ public class AclChangeSets
private Long maxChangeSetId;
AclChangeSets(List<AclChangeSet> aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId)
public AclChangeSets(List<AclChangeSet> aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId)
{
this.aclChangeSets = (aclChangeSets == null ? null : new ArrayList<>(aclChangeSets));
this.maxChangeSetCommitTime = maxChangeSetCommitTime;