From 354ac5cfe3bc9620a5c81242914c98dd29354bd8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2019 22:13:38 +0000 Subject: [PATCH 001/129] Bump alfresco-data-model from 8.53 to 8.54 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.53 to 8.54. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.53...8.54) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 5c75cb1e0..381855471 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.53 + 8.54 2.10.1 From dbfbf2f9e0e4348ac78bc6824d793032cf7fba78 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 20 Nov 2019 10:44:04 +0100 Subject: [PATCH 002/129] [ SEARCH-1861 ] Remove ignored test --- .../alfresco/solr/tracker/AclTrackerTest.java | 282 ------------------ 1 file changed, 282 deletions(-) delete mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclTrackerTest.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclTrackerTest.java deleted file mode 100644 index 1e182dc39..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AclTrackerTest.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ConcurrentLinkedQueue; - -import org.alfresco.solr.IndexTrackingShutdownException; -import org.alfresco.solr.InformationServer; -import org.alfresco.solr.TrackerState; -import org.alfresco.solr.client.AclChangeSet; -import org.alfresco.solr.client.AclChangeSets; -import org.alfresco.solr.client.SOLRAPIClient; -import org.apache.commons.lang.reflect.FieldUtils; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -/** - * Unit tests for the {@link AclTracker} class. - * - * @author Matt Ward - */ -@RunWith(MockitoJUnitRunner.class) -public class AclTrackerTest -{ - private static final Object CUSTOM_ALFRESCO_VERSION = "99.9.9"; - // The class under test - private AclTracker tracker; - private @Mock SOLRAPIClient client; - private @Mock InformationServer informationServer; - private TrackerState trackerState; - - @Before - public void setUp() throws Exception - { - trackerState = new TrackerState(); - trackerState.setRunning(false); // Nothing would happen if it were already running. - Properties props = createProperties(); - // Spy in order to stub some methods and verify others. Note: non-stubbed methods go to the real class. - tracker = spy(new AclTracker(props, client, "core-name", informationServer)); - } - - private Properties createProperties() - { - Properties props = new Properties(); - props.put("alfresco.stores", "workspace://SpacesStore"); - props.put("alfresco.version", CUSTOM_ALFRESCO_VERSION); - return props; - } - - protected void testTrackChangesRan() - { - when(tracker.getTrackerState()).thenReturn(trackerState); - - tracker.track(); - - assertFalse(trackerState.isRunning()); - assertFalse(trackerState.isCheck()); - } - - - // High level test of doTrack() workflow. - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void checkTrackingOperaionsPerformed() throws Throwable - { - testTrackChangesRan(); - - verify(tracker).purgeAclChangeSets(); - verify(tracker).purgeAcls(); - - verify(tracker).reindexAclChangeSets(); - verify(tracker).reindexAcls(); - - verify(tracker).indexAclChangeSets(); - verify(tracker).indexAcls(); - - verify(tracker).trackRepository(); - } - - - // Tests the purgeAclChangeSets() call made in AclTracker.doTrack() - // TODO: the other operations in doTrack(). - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void checkTrackingWhenAclChangeSetsToPurge() throws IllegalAccessException, IOException - { - @SuppressWarnings("unchecked") - ConcurrentLinkedQueue aclChangeSetsToPurge = (ConcurrentLinkedQueue) - FieldUtils.readField(tracker, "aclChangeSetsToPurge", true); - aclChangeSetsToPurge.add(101L); - aclChangeSetsToPurge.add(102L); - aclChangeSetsToPurge.add(103L); - - // Invoke the behaviour we're testing - testTrackChangesRan(); - - // aclChangeSetsToPurge - verify(informationServer).deleteByAclChangeSetId(101L); - verify(informationServer).deleteByAclChangeSetId(102L); - verify(informationServer).deleteByAclChangeSetId(103L); - - // TODO: verify checkShutdown - verify(informationServer).commit(); - } - - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void checkTrackingWhenAclsToPurge() throws IllegalAccessException, IOException - { - @SuppressWarnings("unchecked") - ConcurrentLinkedQueue aclsToPurge = (ConcurrentLinkedQueue) - FieldUtils.readField(tracker, "aclsToPurge", true); - aclsToPurge.add(201L); - aclsToPurge.add(202L); - aclsToPurge.add(203L); - - // Invoke the behaviour we're testing - testTrackChangesRan(); - - // aclsToPurge - verify(informationServer).deleteByAclId(201L); - verify(informationServer).deleteByAclId(202L); - verify(informationServer).deleteByAclId(203L); - - // TODO: verify checkShutdown - verify(informationServer).commit(); - } - - // TODO: Commented out, approach not working - // Rethink or finish later. - /*@Test - public void checkTrackingWhenAclChangeSetsToReindex() throws IllegalAccessException, IOException, AuthenticationException, JSONException - { - @SuppressWarnings("unchecked") - ConcurrentLinkedQueue aclChangeSetsToReindex = (ConcurrentLinkedQueue) - FieldUtils.readField(tracker, "aclChangeSetsToReindex", true); - aclChangeSetsToReindex.add(301L); - aclChangeSetsToReindex.add(302L); - aclChangeSetsToReindex.add(303L); - - // tracker will loop through list of ACLs to reindex, and get changesets for each changeset ID. - when(client.getAclChangeSets(null, 201L, null, 202L, 1)).thenReturn(mockChangeSets(201L)); - when(client.getAclChangeSets(null, 202L, null, 203L, 1)).thenReturn(mockChangeSets(202L)); - when(client.getAclChangeSets(null, 203L, null, 204L, 1)).thenReturn(mockChangeSets(203L)); - - when(client.getAcls(Collections.singletonList(new AclChangeSet(201L, 0L, 1)), null, Integer.MAX_VALUE)).thenReturn(Collections.singletonList(new Acl(201L, 0L))); - when(client.getAcls(Collections.singletonList(new AclChangeSet(202L, 0L, 1)), null, Integer.MAX_VALUE)).thenReturn(Collections.singletonList(new Acl(202L, 0L))); - when(client.getAcls(Collections.singletonList(new AclChangeSet(203L, 0L, 1)), null, Integer.MAX_VALUE)).thenReturn(Collections.singletonList(new Acl(203L, 0L))); - - // TODO: ...more... - - // Invoke the behaviour we're testing - testTrackChangesRan(); - - verify(informationServer).deleteByAclChangeSetId(301L); - verify(informationServer).deleteByAclChangeSetId(302L); - verify(informationServer).deleteByAclChangeSetId(303L); - - // TODO: verify checkShutdown - verify(informationServer).commit(); - }*/ - - private AclChangeSets mockChangeSets(long id) - { - List changeSets = Collections.singletonList(new AclChangeSet(id, 0L, 1)); - AclChangeSets acs = mock(AclChangeSets.class); - when(acs.getAclChangeSets()).thenReturn(changeSets); - return acs; - } - - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void trackingAbortsWhenAlreadyRunning() throws Throwable - { - trackerState.setRunning(true); - // Prove running state, before attempt to track() - assertTrue(trackerState.isRunning()); - - FieldUtils.writeField(tracker, "state", trackerState, true); - tracker.track(); - - // Still running - these values are unaffected. - assertTrue(trackerState.isRunning()); - assertFalse(trackerState.isCheck()); - - // Prove doTrack() was not called - verify(tracker, never()).doTrack(); - } - - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void willRollbackOnThrowableDuringTracking() throws Throwable - { - doThrow(new RuntimeException("Simulated problem during tracking")).when(tracker).doTrack(); - - testTrackChangesRan(); - - verify(informationServer).rollback(); - } - - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void willRollbackOnIndexTrackingShutdownException() throws Throwable - { - doThrow(new IndexTrackingShutdownException()).when(tracker).doTrack(); - - testTrackChangesRan(); - - verify(informationServer).rollback(); - } - - @Ignore("Not yet implemented.") - @Test - public void canCheckIndex() - { - // TODO -// tracker.checkIndex(fromTx, toTx, fromAclTx, toAclTx, fromTime, toTime); - } - - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void canGetAlfrescoVersion() - { - // Check we're testing something useful - assertNotNull(CUSTOM_ALFRESCO_VERSION); - // Check alfresco version retrieved from properties - assertEquals(CUSTOM_ALFRESCO_VERSION, tracker.getAlfrescoVersion()); - } - - - @Test - @Ignore("Superseded by AlfrescoSolrTrackerTest") - public void canClose() throws IllegalAccessException - { - ThreadHandler threadHandler = (ThreadHandler) FieldUtils.readField(tracker, "threadHandler", true); - threadHandler = spy(threadHandler); - FieldUtils.writeField(tracker, "threadHandler", threadHandler, true); - - tracker.shutdown(); - - // AclTracker specific - verify(threadHandler).shutDownThreadPool(); - - // Applicable to all AbstractAclTracker - verify(client).close(); - } -} From 2fbb28123f0e2364f323007766f1b51e40e81f92 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 20 Nov 2019 10:45:46 +0100 Subject: [PATCH 003/129] [ SEARCH-1861 ] Removed commented test method in SolrInformationTest --- .../solr/SolrInformationServerTest.java | 78 +------------------ 1 file changed, 3 insertions(+), 75 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java index 69c910e7f..aeb59b41b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrInformationServerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * @@ -19,18 +19,15 @@ package org.alfresco.solr; -import java.io.IOException; import java.util.Properties; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.content.SolrContentStore; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -43,8 +40,6 @@ import org.mockito.junit.MockitoJUnitRunner; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -99,6 +94,7 @@ public class SolrInformationServerTest request = infoServer.newSolrQueryRequest(); } + @SuppressWarnings("unchecked") @Test public void testGetStateOk() { @@ -124,6 +120,7 @@ public class SolrInformationServerTest /** * GetState returns null in case the given id doesn't correspond to an existing state document. */ + @SuppressWarnings("unchecked") @Test public void testGetStateWithStateNotFound_returnsNull() { @@ -143,73 +140,4 @@ public class SolrInformationServerTest assertNull(document); } - - @Test - public void testIndexAcl() throws IOException - { - /* - // Source/expected data - List aclReadersList = new ArrayList(); - aclReadersList.add(new AclReaders(101, Arrays.asList("r1", "r2", "r3"), Arrays.asList("d1", "d2"), 999, - "example.com")); - aclReadersList.add(new AclReaders(102, Arrays.asList("r4", "r5", "r6"), Arrays.asList("d3", "d4"), 999, - "another.test")); - aclReadersList.add(new AclReaders(103, Arrays.asList("GROUP_marketing", "simpleuser", "GROUP_EVERYONE", - "ROLE_GUEST", "ROLE_ADMINISTRATOR", "ROLE_OWNER", "ROLE_RANDOM"), Arrays.asList( - "GROUP_engineering", "justauser", "GROUP_EVERYONE", "ROLE_GUEST", "ROLE_ADMINISTRATOR", - "ROLE_OWNER", "ROLE_RANDOM"), 999, "tenant.test")); - aclReadersList.add(new AclReaders(104, Arrays.asList("GROUP_marketing", "simpleuser", "GROUP_EVERYONE", - "ROLE_GUEST", "ROLE_ADMINISTRATOR", "ROLE_OWNER", "ROLE_RANDOM"), Arrays.asList( - "GROUP_engineering", "justauser", "GROUP_EVERYONE", "ROLE_GUEST", "ROLE_ADMINISTRATOR", - "ROLE_OWNER", "ROLE_RANDOM"), 999, "" Zero-length tenant == no mangling )); - - //final boolean willOverwrite = true; - - // Invoke the method under test - //infoServer.indexAcl(aclReadersList, willOverwrite); - - - // Capture the AddUpdateCommand instances for further analysis. - //ArgumentCaptor cmdArg = ArgumentCaptor.forClass(AddUpdateCommand.class); - // The processor processes as many add commands as there are items in the aclReaderList. - //verify(processor, times(aclReadersList.size())).processAdd(cmdArg.capture()); - - // Verify that the AddUpdateCommand is as expected. - //List updates = cmdArg.getAllValues(); - //assertEquals("Wrong number of updates", aclReadersList.size(), updates.size()); - /* - for (int docIndex = 0; docIndex < updates.size(); docIndex++) - { - AddUpdateCommand update = updates.get(docIndex); - assertEquals("Overwrite flag was not correct value.", willOverwrite, update.overwrite); - SolrInputDocument inputDoc = update.getSolrInputDocument(); - // Retrieve the original AclReaders object and compare with data in submitted SolrInputDocument - final AclReaders sourceAclReaders = aclReadersList.get(docIndex); - assertEquals( - AlfrescoSolrDataModel.getTenantId(sourceAclReaders.getTenantDomain()) + "!" - + NumericEncoder.encode(sourceAclReaders.getId()) + "!ACL", - inputDoc.getFieldValue("id").toString()); - assertEquals("0", inputDoc.getFieldValue("_version_").toString()); - assertEquals(sourceAclReaders.getId(), inputDoc.getFieldValue(QueryConstants.FIELD_ACLID)); - assertEquals(sourceAclReaders.getAclChangeSetId(), inputDoc.getFieldValue(QueryConstants.FIELD_INACLTXID)); - - if (sourceAclReaders.getId() == 103) - { - // Authorities *may* (e.g. GROUP, EVERYONE, GUEST) be mangled to include tenant information - final Collection docReaders = inputDoc.getFieldValues(QueryConstants.FIELD_READER); - assertEquals(Arrays.asList("GROUP_marketing@tenant.test", "simpleuser", "GROUP_EVERYONE@tenant.test", - "ROLE_GUEST@tenant.test", "ROLE_ADMINISTRATOR", "ROLE_OWNER", "ROLE_RANDOM"), docReaders); - final Collection docDenied = inputDoc.getFieldValues(QueryConstants.FIELD_DENIED); - assertEquals(Arrays.asList("GROUP_engineering@tenant.test", "justauser", "GROUP_EVERYONE@tenant.test", - "ROLE_GUEST@tenant.test", "ROLE_ADMINISTRATOR", "ROLE_OWNER", "ROLE_RANDOM"), docDenied); - } - else - { - // Simple case, no authority/tenant mangling. - assertEquals(sourceAclReaders.getReaders(), inputDoc.getFieldValues(QueryConstants.FIELD_READER)); - assertEquals(sourceAclReaders.getDenied(), inputDoc.getFieldValues(QueryConstants.FIELD_DENIED)); - } - } - */ - } } From b82318204012b5000bcff43930b5c8fb3ebc043c Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 20 Nov 2019 10:46:30 +0100 Subject: [PATCH 004/129] [ SEARCH-1861 ] Remove alfresco.version --- .../alfresco/solr/HandlerReportHelper.java | 10 ---- .../alfresco/solr/SolrInformationServer.java | 57 ++++++++----------- .../solr/tracker/AbstractTracker.java | 52 ++++++----------- .../solr/tracker/SlaveCoreStatePublisher.java | 18 ++++++ .../org/alfresco/solr/tracker/Tracker.java | 2 - 5 files changed, 61 insertions(+), 78 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java index a3c2b069d..72c3e27c0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java @@ -145,9 +145,6 @@ class HandlerReportHelper return payload; } - /** - * Builds Tracker report - */ static NamedList buildTrackerReport(TrackerRegistry trackerRegistry, InformationServer srv, String coreName, Long fromTx, Long toTx, Long fromAclTx, Long toAclTx, Long fromTime, Long toTime) throws JSONException { @@ -157,7 +154,6 @@ class HandlerReportHelper AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime); NamedList ihr = new SimpleOrderedMap<>(); - ihr.add("Alfresco version", aclTracker.getAlfrescoVersion()); ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() .cardinality()); @@ -277,12 +273,6 @@ class HandlerReportHelper Date end = new Date(now.getTime() + remainingTxTimeMillis); Duration remainingTx = new Duration(now, end); - long remainingChangeSetTimeMillis = 0; - - now = new Date(); - end = new Date(now.getTime() + remainingChangeSetTimeMillis); - Duration remainingChangeSet = new Duration(now, end); - NamedList ftsSummary = new SimpleOrderedMap<>(); long remainingContentTimeMillis = 0; srv.addFTSStatusCounts(ftsSummary); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 189a5cf8f..b1f8e871a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 Alfresco Software Limited. + * Copyright (C) 2019 Alfresco Software Limited. * * This file is part of Alfresco * @@ -178,14 +178,14 @@ import org.springframework.extensions.surf.util.I18NUtil; import org.springframework.util.FileCopyUtils; /** - * This is the Solr4 implementation of the information server (index). + * This is the Apache Solr implementation of the information server (index). + * * @author Ahmed Owian * @since 5.0 */ public class SolrInformationServer implements InformationServer { private final static Log LOGGER = new Log(SolrInformationServer.class); - private final static long TWO_MINUTES = 120000; private static final String NO_SITE = "_REPOSITORY_"; private static final String SHARED_FILES = "_SHARED_FILES_"; @@ -200,23 +200,23 @@ public class SolrInformationServer implements InformationServer public static final String AND = " AND "; public static final String OR = " OR "; - public static final String REQUEST_HANDLER_ALFRESCO_FULL_TEXT_SEARCH = "/afts"; - public static final String REQUEST_HANDLER_NATIVE = "/native"; - public static final String REQUEST_HANDLER_ALFRESCO = "/alfresco"; - public static final String REQUEST_HANDLER_SELECT = "/select"; - public static final String REQUEST_HANDLER_GET = "/get"; - public static final String RESPONSE_DEFAULT_IDS = "response"; - public static final String RESPONSE_DEFAULT_ID = "doc"; + //public static final String REQUEST_HANDLER_ALFRESCO_FULL_TEXT_SEARCH = "/afts"; + private static final String REQUEST_HANDLER_NATIVE = "/native"; + //public static final String REQUEST_HANDLER_ALFRESCO = "/alfresco"; + //public static final String REQUEST_HANDLER_SELECT = "/select"; + static final String REQUEST_HANDLER_GET = "/get"; + private static final String RESPONSE_DEFAULT_IDS = "response"; + static final String RESPONSE_DEFAULT_ID = "doc"; - public static final String PREFIX_ERROR = "ERROR-"; + static final String PREFIX_ERROR = "ERROR-"; public static final String DOC_TYPE_NODE = "Node"; - public static final String DOC_TYPE_UNINDEXED_NODE = "UnindexedNode"; - public static final String DOC_TYPE_ERROR_NODE = "ErrorNode"; + private static final String DOC_TYPE_UNINDEXED_NODE = "UnindexedNode"; + private static final String DOC_TYPE_ERROR_NODE = "ErrorNode"; public static final String DOC_TYPE_ACL = "Acl"; public static final String DOC_TYPE_TX = "Tx"; public static final String DOC_TYPE_ACL_TX = "AclTx"; - public static final String DOC_TYPE_STATE = "State"; + private static final String DOC_TYPE_STATE = "State"; public static final String SOLR_HOST = "solr.host"; public static final String SOLR_PORT = "solr.port"; @@ -235,7 +235,6 @@ public class SolrInformationServer implements InformationServer private final TrackerStats trackerStats = new TrackerStats(this); private final AlfrescoSolrDataModel dataModel; private final SolrContentStore solrContentStore; - private final String alfrescoVersion; private final boolean transformContent; private final boolean recordUnindexedNodes; private final long lag; @@ -271,7 +270,7 @@ public class SolrInformationServer implements InformationServer protected enum FTSStatus {New, Dirty, Clean} - class DocListCollector implements Collector, LeafCollector + static class DocListCollector implements Collector, LeafCollector { private IntArrayList docs = new IntArrayList(); private int docBase; @@ -303,7 +302,7 @@ public class SolrInformationServer implements InformationServer } } - class TxnCacheFilter extends DelegatingCollector + static class TxnCacheFilter extends DelegatingCollector { private NumericDocValues currentLongs; private Map txnLRU; @@ -330,7 +329,7 @@ public class SolrInformationServer implements InformationServer } } - class TxnCollector extends DelegatingCollector + static class TxnCollector extends DelegatingCollector { private NumericDocValues currentLongs; private long txnFloor; @@ -372,7 +371,7 @@ public class SolrInformationServer implements InformationServer } } - class LRU extends LinkedHashMap + static class LRU extends LinkedHashMap { private int maxSize; @@ -395,7 +394,7 @@ public class SolrInformationServer implements InformationServer boolean isDefinitionExists(QName qName); } - abstract class TransactionInfoReporter + static abstract class TransactionInfoReporter { protected final IndexHealthReport report; @@ -428,7 +427,6 @@ public class SolrInformationServer implements InformationServer this.solrContentStore = solrContentStore; Properties p = core.getResourceLoader().getCoreProperties(); - alfrescoVersion = p.getProperty("alfresco.version", "Unknown"); transformContent = Boolean.parseBoolean(p.getProperty("alfresco.index.transformContent", "true")); recordUnindexedNodes = Boolean.parseBoolean(p.getProperty("alfresco.recordUnindexedNodes", "true")); lag = Integer.parseInt(p.getProperty("alfresco.lag", "1000")); @@ -505,11 +503,6 @@ public class SolrInformationServer implements InformationServer } } - public String getAlfrescoVersion() - { - return this.alfrescoVersion; - } - @Override public void afterInitModels() { @@ -1547,7 +1540,7 @@ public class SolrInformationServer implements InformationServer StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_INDEXED); if (pValue != null) { - boolean isIndexed = Boolean.valueOf(pValue.getValue()); + boolean isIndexed = Boolean.parseBoolean(pValue.getValue()); if (!isIndexed) { LOGGER.debug("Clearing unindexed"); @@ -1902,7 +1895,7 @@ public class SolrInformationServer implements InformationServer StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_INDEXED); if (pValue != null) { - boolean isIndexed = Boolean.valueOf(pValue.getValue()); + boolean isIndexed = Boolean.parseBoolean(pValue.getValue()); if (!isIndexed) { LOGGER.debug("Clearing unindexed"); @@ -1958,7 +1951,7 @@ public class SolrInformationServer implements InformationServer } } - private void addToNewDocAndCache(NodeMetaData nodeMetaData, SolrInputDocument newDoc) throws IOException + private void addToNewDocAndCache(NodeMetaData nodeMetaData, SolrInputDocument newDoc) { addFieldsToDoc(nodeMetaData, newDoc); SolrInputDocument cachedDoc = null; @@ -2232,7 +2225,7 @@ public class SolrInformationServer implements InformationServer .get(ContentModel.PROP_IS_CONTENT_INDEXED); if (pValue != null) { - boolean isIndexed = Boolean.valueOf(pValue.getValue()); + boolean isIndexed = Boolean.parseBoolean(pValue.getValue()); if (!isIndexed) { isContentIndexed = false; @@ -2393,7 +2386,7 @@ public class SolrInformationServer implements InformationServer if(cachedDoc.getFieldValue(fldName) != null) { - long cachedDocContentDocid = Long.valueOf(String.valueOf(cachedDoc.getFieldValue(fldName))); + long cachedDocContentDocid = Long.parseLong(String.valueOf(cachedDoc.getFieldValue(fldName))); long currentContentDocid = contentPropertyValue.getId(); // If we have used out of date content we mark it as dirty // Otherwise we leave it alone - it could already be marked as dirty/New and require an update @@ -3584,7 +3577,7 @@ public class SolrInformationServer implements InformationServer field, 1); // Min count of 1 ensures that the id returned is in the index for (Map.Entry idCount : idCounts) { - long idInIndex = Long.valueOf(idCount.getKey()); + long idInIndex = Long.parseLong(idCount.getKey()); // Only looks at facet values that fit the query if (batchStartId <= idInIndex && idInIndex <= batchEndId) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index d4429f92b..f4d381cd1 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2019 Alfresco Software Limited. * * This file is part of Alfresco * @@ -18,6 +18,8 @@ */ package org.alfresco.solr.tracker; +import static java.util.Optional.ofNullable; + import java.lang.invoke.MethodHandles; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -49,11 +51,10 @@ public abstract class AbstractTracker implements Tracker protected SOLRAPIClient client; InformationServer infoSrv; protected String coreName; - protected StoreRef storeRef; - protected long batchCount; - protected String alfrescoVersion; - protected TrackerStats trackerStats; - protected boolean runPostModelLoadInit = true; + StoreRef storeRef; + long batchCount; + TrackerStats trackerStats; + boolean runPostModelLoadInit = true; private int maxLiveSearchers; private volatile boolean shutdown = false; @@ -63,9 +64,9 @@ public abstract class AbstractTracker implements Tracker protected volatile TrackerState state; protected int shardCount; protected int shardInstance; - protected String shardMethod; + String shardMethod; protected boolean transformContent; - protected String shardTemplate; + String shardTemplate; protected volatile boolean rollback; protected final Type type; @@ -102,12 +103,8 @@ public abstract class AbstractTracker implements Tracker transformContent = Boolean.parseBoolean(p.getProperty("alfresco.index.transformContent", "true")); this.trackerStats = this.infoSrv.getTrackerStats(); - - alfrescoVersion = p.getProperty("alfresco.version", "5.0.0"); this.type = type; - - LOGGER.info("Solr built for Alfresco version: {}", alfrescoVersion); } @@ -183,12 +180,9 @@ public abstract class AbstractTracker implements Tracker if(this.state == null) { - /* - * Set the global state for the tracker here. - */ this.state = getTrackerState(); - LOGGER.debug("##### Setting tracker global state."); - LOGGER.debug("State set: {}", this.state.toString()); + + LOGGER.debug("Global Tracker State set to: {}", this.state.toString()); this.state.setRunning(true); } else @@ -237,12 +231,11 @@ public abstract class AbstractTracker implements Tracker finally { infoSrv.unregisterTrackerThread(); - if(state != null) - { - //During a rollback state is set to null. + ofNullable(state).ifPresent(tstate -> { + // During a rollback state is set to null. state.setRunning(false); state.setCheck(false); - } + }); runLock.release(); } } @@ -284,7 +277,7 @@ public abstract class AbstractTracker implements Tracker /** * Allows time for the scheduled asynchronous tasks to complete */ - protected synchronized void waitForAsynchronous() + synchronized void waitForAsynchronous() { AbstractWorkerRunnable currentRunnable = this.threadHandler.peekHeadReindexWorker(); while (currentRunnable != null) @@ -305,12 +298,12 @@ public abstract class AbstractTracker implements Tracker } } - public int getMaxLiveSearchers() + int getMaxLiveSearchers() { return maxLiveSearchers; } - protected void checkShutdown() + void checkShutdown() { if(shutdown) { @@ -345,20 +338,11 @@ public abstract class AbstractTracker implements Tracker return this.writeLock; } - public Semaphore getRunLock() + Semaphore getRunLock() { return this.runLock; } - /** - * @return Alfresco version Solr was built for - */ - @Override - public String getAlfrescoVersion() - { - return alfrescoVersion; - } - public Properties getProps() { return props; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 19a93631c..71750c53f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ package org.alfresco.solr.tracker; import static org.alfresco.solr.tracker.Tracker.Type.NODE_STATE_PUBLISHER; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java index 05fe0766a..976d52c20 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java @@ -32,8 +32,6 @@ public interface Tracker boolean hasMaintenance() throws Exception; Semaphore getWriteLock(); - - String getAlfrescoVersion(); void setShutdown(boolean shutdown); From 1f9bcc55752ec6752c06d46015d8b60fba9677cb Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 20 Nov 2019 10:47:18 +0100 Subject: [PATCH 005/129] [ SEARCH-1861 ] Removed alfresco.version property --- .../java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java index e685b14be..a5a0d1ab4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/SolrTrackerSchedulerTest.java @@ -78,7 +78,6 @@ public class SolrTrackerSchedulerTest props.put("shard.method", "SHARD_METHOD_DBID"); props.put("alfresco.template", ""); props.put("alfresco.index.transformContent", "true"); - props.put("alfresco.version", "5.0.0"); } @After From f830d6409f1e8ef3ed6951f7f87428954fbadfb1 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 20 Nov 2019 10:48:22 +0100 Subject: [PATCH 006/129] [ SEARCH-1861 ] alfresco.version removed from Content Tracker ADR --- .../doc/architecture/trackers/00001-content-tracker.md | 1 - 1 file changed, 1 deletion(-) diff --git a/search-services/alfresco-search/doc/architecture/trackers/00001-content-tracker.md b/search-services/alfresco-search/doc/architecture/trackers/00001-content-tracker.md index e267a79a9..0387374eb 100644 --- a/search-services/alfresco-search/doc/architecture/trackers/00001-content-tracker.md +++ b/search-services/alfresco-search/doc/architecture/trackers/00001-content-tracker.md @@ -155,7 +155,6 @@ The following table illustrates the configuration properties used by the Tracker |shard.method|"DB_ID"|Data (Documents, ACLs) Routing criteria among shards| | |Y|Y| | | |alfresco.fingerprint|true|true if we want to compute the content Fingerprint| | |Y|| | | |alfresco.index.transformContent|true| | | |Y|| | | -|alfresco.version|5.0.0|The target Alfresco version| | | | | | | |alfresco.corePoolSize|4|The number of threads to keep in the pool, even if they are idle|Y|Y|Y|Y|Y|Y| |alfresco.maximumPoolSize|-1|The maximum number of threads allowed in the pool|Y|Y|Y|Y|Y|Y| |alfresco.keepAliveTime|120|When the number of threads is greater than the core pool size, this is the maximum time that excess idle threads will wait for new tasks before terminating|Y|Y|Y|Y|Y|Y| From bad3520d9c78c5e67b92d3e03772c4e381908c27 Mon Sep 17 00:00:00 2001 From: mbhave Date: Thu, 21 Nov 2019 17:56:57 +0000 Subject: [PATCH 007/129] Search-1313: Test added to test fix for the MNT --- .../SolrSearchSecondaryAssociationTests.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java new file mode 100644 index 000000000..d69fadb74 --- /dev/null +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java @@ -0,0 +1,70 @@ +package org.alfresco.test.search.functional.searchServices.search; + +import org.alfresco.rest.model.RestNodeChildAssociationModel; +import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; +import org.alfresco.utility.data.CustomObjectTypeProperties; +import org.alfresco.utility.model.FileModel; +import org.alfresco.utility.model.FolderModel; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Test class tests content in the secondary parent is found too + * Created for Search-1313 + * + * @author Meenal Bhave + */ +public class SolrSearchSecondaryAssociationTests extends AbstractE2EFunctionalTest +{ + private FolderModel testFolder1, testFolder2; + private FileModel file1; + + @BeforeClass(alwaysRun = true) + public void dataPreparation() throws Exception + { + // Folders + testFolder1 = new FolderModel("folder1"); + testFolder2 = new FolderModel("folder2"); + + // File(s) + file1 = new FileModel("file1.txt"); + file1.setContent("content file 1"); + + // Create folder1 + dataContent.usingUser(testUser).usingSite(testSite).createCustomContent(testFolder1, "cmis:folder", new CustomObjectTypeProperties()); + + // Create file1 + dataContent.usingUser(testUser).usingResource(testFolder1).createCustomContent(file1, "cmis:document", new CustomObjectTypeProperties()); + + // Create folder2 + dataContent.usingUser(testUser).usingSite(testSite).createCustomContent(testFolder2, "cmis:folder", new CustomObjectTypeProperties()); + + // Create Secondary association + RestNodeChildAssociationModel childAssoc1 = new RestNodeChildAssociationModel(file1.getNodeRefWithoutVersion(), "cm:contains"); + String secondaryChildrenBody = "[" + childAssoc1.toJson() + "]"; + + restClient.authenticateUser(testUser).withCoreAPI().usingResource(testFolder2).createSecondaryChildren(secondaryChildrenBody); + + // wait for solr index + waitForMetadataIndexing(file1.getName(), true); + } + + @Test(priority = 1) + public void testPathForSecondaryAssociation() + { + String queryPathFolder1 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + + "/cm:documentLibrary/cm:" + testFolder1.getName() + "/cm:" + file1.getName() + "\""; + + // Test if file can be found in Primary Parent + boolean found = isContentInSearchResults(queryPathFolder1, file1.getName(), true); + Assert.assertTrue(found, "File found using Primary Parent Path"); + + String queryPathFolder2 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + + "/cm:documentLibrary/cm:" + testFolder2.getName() + "/cm:" + file1.getName() + "\""; + + // Test if file can be found in Secondary Parent + found = isContentInSearchResults(queryPathFolder2, file1.getName(), true); + Assert.assertTrue(found, "File found using Secondary Parent Path"); + } +} From 8ac3298ef903314fe0c81514d7aa9156c32da198 Mon Sep 17 00:00:00 2001 From: mbhave Date: Thu, 21 Nov 2019 17:58:49 +0000 Subject: [PATCH 008/129] Class renamed for consistency --- ...sociationTests.java => SearchSecondaryAssociationTests.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/{SolrSearchSecondaryAssociationTests.java => SearchSecondaryAssociationTests.java} (97%) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTests.java similarity index 97% rename from e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java rename to e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTests.java index d69fadb74..81a5da5f0 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SolrSearchSecondaryAssociationTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTests.java @@ -15,7 +15,7 @@ import org.testng.annotations.Test; * * @author Meenal Bhave */ -public class SolrSearchSecondaryAssociationTests extends AbstractE2EFunctionalTest +public class SearchSecondaryAssociationTests extends AbstractE2EFunctionalTest { private FolderModel testFolder1, testFolder2; private FileModel file1; From b1d210d44934604a5dac3e4e273dc20baee0fb12 Mon Sep 17 00:00:00 2001 From: mbhave Date: Thu, 21 Nov 2019 17:59:49 +0000 Subject: [PATCH 009/129] Class renamed for consistency --- ...ssociationTests.java => SearchSecondaryAssociationTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/{SearchSecondaryAssociationTests.java => SearchSecondaryAssociationTest.java} (97%) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java similarity index 97% rename from e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTests.java rename to e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java index 81a5da5f0..fe68c0c7a 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java @@ -15,7 +15,7 @@ import org.testng.annotations.Test; * * @author Meenal Bhave */ -public class SearchSecondaryAssociationTests extends AbstractE2EFunctionalTest +public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest { private FolderModel testFolder1, testFolder2; private FileModel file1; From d7d7e7234b925dd5c8eb0b7e3500888e8f2728df Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 22 Nov 2019 09:15:51 +0100 Subject: [PATCH 010/129] [ SEARCH-1313 ] stream filtering using anyMatch() --- .../functional/AbstractE2EFunctionalTest.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index 7470000ab..705fb848d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -279,18 +279,17 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont } /** - * Method to check if the contentName is returned in the SearchResponse - * @param response - * @param contentName - * @return + * Method to check if the contentName is returned in the SearchResponse. + * + * @param response the search response + * @param contentName the text we are using as matching/verifying criteria. + * @return true if if the item with the contentName text is returned in the SearchResponse. */ public boolean isContentInSearchResponse(SearchResponse response, String contentName) { - boolean found = response.getEntries().stream() + return response.getEntries().stream() .map(entry -> entry.getModel().getName()) - .filter(name -> name.equalsIgnoreCase(contentName) || contentName.isBlank()).count() > 0; - - return found; + .anyMatch(name -> name.equalsIgnoreCase(contentName) || contentName.isBlank()); } /** @@ -426,8 +425,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Helper method to test if the search query works and count matches where provided * @param query: AFTS or cmis query string - * @param expectedCount: Only successful response is checked, when expectedCount is null (can not be exactly specified), - * @param setCmis: Query language is set to cmis when setCmis is true, AFTS when false + * @param expectedCount: Only successful response is checked, when expectedCount is null (can not be exactly specified), * @return SearchResponse */ protected SearchResponse testSearchQuery(String query, Integer expectedCount, SearchLanguage queryLanguage) From 507e75bc750bbab64750a1181d076a9930444f15 Mon Sep 17 00:00:00 2001 From: mbhave Date: Fri, 22 Nov 2019 10:48:16 +0000 Subject: [PATCH 011/129] Improved test to include removal of secondary association --- .../SearchSecondaryAssociationTest.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java index fe68c0c7a..faaab5a47 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java @@ -1,5 +1,6 @@ package org.alfresco.test.search.functional.searchServices.search; +import org.alfresco.rest.model.RestNodeAssociationModelCollection; import org.alfresco.rest.model.RestNodeChildAssociationModel; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.data.CustomObjectTypeProperties; @@ -21,7 +22,7 @@ public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest private FileModel file1; @BeforeClass(alwaysRun = true) - public void dataPreparation() throws Exception + public void dataPreparation() { // Folders testFolder1 = new FolderModel("folder1"); @@ -40,31 +41,44 @@ public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest // Create folder2 dataContent.usingUser(testUser).usingSite(testSite).createCustomContent(testFolder2, "cmis:folder", new CustomObjectTypeProperties()); - // Create Secondary association - RestNodeChildAssociationModel childAssoc1 = new RestNodeChildAssociationModel(file1.getNodeRefWithoutVersion(), "cm:contains"); - String secondaryChildrenBody = "[" + childAssoc1.toJson() + "]"; - - restClient.authenticateUser(testUser).withCoreAPI().usingResource(testFolder2).createSecondaryChildren(secondaryChildrenBody); - // wait for solr index waitForMetadataIndexing(file1.getName(), true); } @Test(priority = 1) - public void testPathForSecondaryAssociation() + public void testSearchPathForSecondaryAssociation() throws Exception { String queryPathFolder1 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + "/cm:documentLibrary/cm:" + testFolder1.getName() + "/cm:" + file1.getName() + "\""; - // Test if file can be found in Primary Parent + // Test if file can be found in folder1: Primary Parent boolean found = isContentInSearchResults(queryPathFolder1, file1.getName(), true); - Assert.assertTrue(found, "File found using Primary Parent Path"); + Assert.assertTrue(found, "File Not found using Primary Parent Path"); String queryPathFolder2 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + "/cm:documentLibrary/cm:" + testFolder2.getName() + "/cm:" + file1.getName() + "\""; - // Test if file can be found in Secondary Parent - found = isContentInSearchResults(queryPathFolder2, file1.getName(), true); + // Test if file can not be found in folder2 + found = isContentInSearchResults(queryPathFolder2, file1.getName(), false); Assert.assertTrue(found, "File found using Secondary Parent Path"); + + // Create Secondary association in folder2 + RestNodeChildAssociationModel childAssoc1 = new RestNodeChildAssociationModel(file1.getNodeRefWithoutVersion(), "cm:contains"); + String secondaryChildrenBody = "[" + childAssoc1.toJson() + "]"; + + restClient.authenticateUser(testUser).withCoreAPI().usingResource(testFolder2).createSecondaryChildren(secondaryChildrenBody); + RestNodeAssociationModelCollection secondaryChildren = restClient.authenticateUser(testUser).withCoreAPI().usingResource(testFolder2).getSecondaryChildren(); + secondaryChildren.getEntryByIndex(0).assertThat().field("id").is(file1.getNodeRefWithoutVersion()); + + // Test if file can be found in folder2: Secondary Parent + found = isContentInSearchResults(queryPathFolder2, file1.getName(), true); + Assert.assertTrue(found, "File Not found using Secondary Parent Path"); + + // Remove Secondary association + restClient.authenticateUser(testUser).withCoreAPI().usingResource(testFolder2).deleteSecondaryChild(secondaryChildren.getEntryByIndex(0)); + + // Test if file can not be found in folder2 + found = isContentInSearchResults(queryPathFolder2, file1.getName(), false); + Assert.assertTrue(found, "File found using Secondary Parent Path"); } } From e1ac8148956a15b29c370ffd435f7fd77a9e6342 Mon Sep 17 00:00:00 2001 From: mbhave Date: Fri, 22 Nov 2019 12:21:04 +0000 Subject: [PATCH 012/129] Search-1313: Added copyright info --- .../search/SearchSecondaryAssociationTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java index faaab5a47..b382bc6f3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java @@ -1,3 +1,10 @@ +/* + * Copyright 2019 Alfresco Software, Ltd. All rights reserved. + * License rights for this program may be obtained from Alfresco Software, Ltd. + * pursuant to a written agreement and any use of this program without such an + * agreement is prohibited. + */ + package org.alfresco.test.search.functional.searchServices.search; import org.alfresco.rest.model.RestNodeAssociationModelCollection; From 73ce8a36ba0b82a60b9f0a5d7ca8464abb2e884e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2019 22:12:43 +0000 Subject: [PATCH 013/129] Bump alfresco-data-model from 8.54 to 8.55 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.54 to 8.55. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.54...8.55) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 381855471..811e1893e 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.54 + 8.55 2.10.1 From e2420abce27c43ef875f4011bae5c78ce4de990d Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 26 Nov 2019 11:36:50 +0000 Subject: [PATCH 014/129] SEARCH-1973 Upgrade to cxf-* 3.2.5. --- search-services/alfresco-search/pom.xml | 51 +++++++++++++++++++ search-services/packaging/pom.xml | 4 +- .../src/main/resources/licenses/notice.txt | 26 +++++----- search-services/pom.xml | 1 + 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 8cdd7d1f7..0be13e026 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -96,6 +96,57 @@ 2.3.2 + + org.apache.cxf + cxf-core + ${cxf.version} + + + org.apache.cxf + cxf-rt-bindings-soap + ${cxf.version} + + + org.apache.cxf + cxf-rt-bindings-xml + ${cxf.version} + + + org.apache.cxf + cxf-rt-databinding-jaxb + ${cxf.version} + + + org.apache.cxf + cxf-rt-frontend-jaxws + ${cxf.version} + + + org.apache.cxf + cxf-rt-frontend-simple + ${cxf.version} + + + org.apache.cxf + cxf-rt-transports-http + ${cxf.version} + + + org.apache.cxf + cxf-rt-ws-addr + ${cxf.version} + + + org.apache.cxf + cxf-rt-ws-policy + ${cxf.version} + + + org.apache.cxf + cxf-rt-wsdl + ${cxf.version} + + junit diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index b210f0634..bf970d553 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -111,7 +111,7 @@ ${project.version} libs ${project.build.directory}/solr-libs - **/jackson-dataformat-smile-*.jar,**/asm-3.3.1.jar,**/jackson-core-asl-*.jar,**/jackson-mapper-asl-*.jar,**/dom4j-1.6.1.jar,**/annotations-1.0.0.jar + **/jackson-dataformat-smile-*.jar,**/asm-3.3.1.jar,**/jackson-core-asl-*.jar,**/jackson-mapper-asl-*.jar,**/dom4j-1.6.1.jar,**/annotations-1.0.0.jar,**/woodstox-core-asl-4.4.1.jar @@ -157,7 +157,9 @@ + + diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 92c61b013..c23912a58 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -32,24 +32,24 @@ json-20160212.jar http://code.google.com/p/json-simple/ === Apache 2.0 === xml-resolver-1.2.jar https://github.com/FasterXML/jackson -neethi-3.0.3.jar http://ws.apache.org/commons/neethi/ +neethi-3.1.1.jar http://ws.apache.org/commons/neethi/ commons-logging-1.2.jar http://jakarta.apache.org/commons/ commons-lang3-3.9.jar http://jakarta.apache.org/commons/ mybatis-3.3.0.jar http://www.mybatis.org/ chemistry-opencmis-commons-impl-1.1.0.jar http://chemistry.apache.org/ chemistry-opencmis-commons-api-1.1.0.jar http://chemistry.apache.org/ -xmlschema-core-2.2.1.jar http://ws.apache.org/commons/XmlSchema/ +xmlschema-core-2.2.3.jar http://ws.apache.org/commons/XmlSchema/ HikariCP-java7-2.4.13.jar https://github.com/brettwooldridge/HikariCP -cxf-core-3.0.12.jar https://cxf.apache.org/ -cxf-rt-bindings-soap-3.0.12.jar https://cxf.apache.org/ -cxf-rt-bindings-xml-3.0.12.jar https://cxf.apache.org/ -cxf-rt-databinding-jaxb-3.0.12.jar https://cxf.apache.org/ -cxf-rt-frontend-jaxws-3.0.12.jar https://cxf.apache.org/ -cxf-rt-frontend-simple-3.0.12.jar https://cxf.apache.org/ -cxf-rt-transports-http-3.0.12.jar https://cxf.apache.org/ -cxf-rt-ws-addr-3.0.12.jar https://cxf.apache.org/ -cxf-rt-ws-policy-3.0.12.jar https://cxf.apache.org/ -cxf-rt-wsdl-3.0.12.jar https://cxf.apache.org/ +cxf-core-3.2.5.jar https://cxf.apache.org/ +cxf-rt-bindings-soap-3.2.5.jar https://cxf.apache.org/ +cxf-rt-bindings-xml-3.2.5.jar https://cxf.apache.org/ +cxf-rt-databinding-jaxb-3.2.5.jar https://cxf.apache.org/ +cxf-rt-frontend-jaxws-3.2.5.jar https://cxf.apache.org/ +cxf-rt-frontend-simple-3.2.5.jar https://cxf.apache.org/ +cxf-rt-transports-http-3.2.5.jar https://cxf.apache.org/ +cxf-rt-ws-addr-3.2.5.jar https://cxf.apache.org/ +cxf-rt-ws-policy-3.2.5.jar https://cxf.apache.org/ +cxf-rt-wsdl-3.2.5.jar https://cxf.apache.org/ mybatis-spring-1.2.5.jar http://www.mybatis.org/ chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/ chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/ @@ -87,6 +87,7 @@ jetty-servlets-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html jetty-util-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html jetty-webapp-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html jetty-xml-9.3.27.v20190418.jar https://www.eclipse.org/jetty/licenses.html +woodstox-core-5.0.3.jar https://github.com/FasterXML/woodstox === CDDL 1.0 === @@ -229,7 +230,6 @@ tika-parsers-1.16.jar tika-xmp-1.16.jar vorbis-java-core-0.8.jar vorbis-java-tika-0.8.jar -woodstox-core-asl-4.4.1.jar xmlbeans-2.6.0.jar xmpcore-5.1.2.jar xz-1.6.jar diff --git a/search-services/pom.xml b/search-services/pom.xml index 8af1e3219..364979b2c 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -15,6 +15,7 @@ Alfresco Solr Search parent 1.7.29 + 3.2.5 alfresco-solrclient-lib From 0bda0f928cb08360f70de04045646d5bec9fcdba Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 27 Nov 2019 08:28:19 +0000 Subject: [PATCH 015/129] SEARCH-1974 SEARCH-1975 Upgrade to Solr 6.6.5-patched.2. This includes later versions of commons-fileupload and bcprov. --- pom.xml | 2 +- .../src/main/resources/licenses/notice.txt | 60 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 8863b968c..9af7a2537 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 11 6.6.5 - ${solr.base.version}-patched.1 + ${solr.base.version}-patched.2 diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index c23912a58..6ebd88fb5 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -116,8 +116,8 @@ asm-commons-5.1.jar aspectjrt-1.8.0.jar attributes-binder-1.3.1.jar avatica-core-1.9.0.jar -bcmail-jdk15-1.45.jar -bcprov-jdk15-1.45.jar +bcmail-jdk15on-1.47.jar +bcprov-jdk15on-1.47.jar boilerpipe-1.1.0.jar caffeine-2.4.0.jar calcite-core-1.11.0.jar @@ -132,7 +132,7 @@ commons-compiler-2.7.6.jar commons-compress-1.14.jar commons-configuration-1.6.jar commons-exec-1.3.jar -commons-fileupload-1.3.2.jar +commons-fileupload-1.3.3.jar commons-io-2.5.jar commons-lang-2.6.jar commons-math3-3.4.1.jar @@ -169,28 +169,28 @@ jul-to-slf4j-1.7.7.jar juniversalchardet-1.0.3.jar langdetect-1.1-20120112.jar log4j-1.2.17.jar -lucene-analyzers-common-6.6.5-patched.1.jar -lucene-analyzers-icu-6.6.5-patched.1.jar -lucene-analyzers-kuromoji-6.6.5-patched.1.jar -lucene-analyzers-morfologik-6.6.5-patched.1.jar -lucene-analyzers-phonetic-6.6.5-patched.1.jar -lucene-analyzers-smartcn-6.6.5-patched.1.jar -lucene-analyzers-stempel-6.6.5-patched.1.jar -lucene-backward-codecs-6.6.5-patched.1.jar -lucene-classification-6.6.5-patched.1.jar -lucene-codecs-6.6.5-patched.1.jar -lucene-core-6.6.5-patched.1.jar -lucene-expressions-6.6.5-patched.1.jar -lucene-grouping-6.6.5-patched.1.jar -lucene-highlighter-6.6.5-patched.1.jar -lucene-join-6.6.5-patched.1.jar -lucene-memory-6.6.5-patched.1.jar -lucene-misc-6.6.5-patched.1.jar -lucene-queries-6.6.5-patched.1.jar -lucene-queryparser-6.6.5-patched.1.jar -lucene-sandbox-6.6.5-patched.1.jar -lucene-spatial-extras-6.6.5-patched.1.jar -lucene-suggest-6.6.5-patched.1.jar +lucene-analyzers-common-6.6.5-patched.2.jar +lucene-analyzers-icu-6.6.5-patched.2.jar +lucene-analyzers-kuromoji-6.6.5-patched.2.jar +lucene-analyzers-morfologik-6.6.5-patched.2.jar +lucene-analyzers-phonetic-6.6.5-patched.2.jar +lucene-analyzers-smartcn-6.6.5-patched.2.jar +lucene-analyzers-stempel-6.6.5-patched.2.jar +lucene-backward-codecs-6.6.5-patched.2.jar +lucene-classification-6.6.5-patched.2.jar +lucene-codecs-6.6.5-patched.2.jar +lucene-core-6.6.5-patched.2.jar +lucene-expressions-6.6.5-patched.2.jar +lucene-grouping-6.6.5-patched.2.jar +lucene-highlighter-6.6.5-patched.2.jar +lucene-join-6.6.5-patched.2.jar +lucene-memory-6.6.5-patched.2.jar +lucene-misc-6.6.5-patched.2.jar +lucene-queries-6.6.5-patched.2.jar +lucene-queryparser-6.6.5-patched.2.jar +lucene-sandbox-6.6.5-patched.2.jar +lucene-spatial-extras-6.6.5-patched.2.jar +lucene-suggest-6.6.5-patched.2.jar metadata-extractor-2.9.1.jar metrics-core-3.2.2.jar metrics-ganglia-3.2.2.jar @@ -214,11 +214,11 @@ rome-1.5.1.jar simple-xml-2.7.1.jar slf4j-api-1.7.7.jar slf4j-log4j12-1.7.7.jar -solr-analysis-extras-6.6.5-patched.1.jar -solr-clustering-6.6.5-patched.1.jar -solr-core-6.6.5-patched.1.jar -solr-langid-6.6.5-patched.1.jar -solr-solrj-6.6.5-patched.1.jar +solr-analysis-extras-6.6.5-patched.2.jar +solr-clustering-6.6.5-patched.2.jar +solr-core-6.6.5-patched.2.jar +solr-langid-6.6.5-patched.2.jar +solr-solrj-6.6.5-patched.2.jar spatial4j-0.6.jar start.jar stax2-api-3.1.4.jar From 422405f6ac81e1f08300cc880106d7c0bbd4525d Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 28 Nov 2019 14:21:39 +0000 Subject: [PATCH 016/129] SEARCH-1986 Update E2E tests to use latest TAS artifacts. --- e2e-test/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 5cee7b842..195de617a 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,8 +11,8 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 6.0.1.2 - 6.0.0.4 + 1.15 + 1.6 3.0.14 3.2.0 src/test/resources/SearchSuite.xml @@ -44,7 +44,7 @@ org.alfresco.tas - restapi-test + restapi ${tas.rest.api.version} test @@ -56,7 +56,7 @@ org.alfresco.tas - cmis-test + cmis ${tas.cmis.api.version} test From b39f0b625c19b58e2399fc0221e4f9121b52689f Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 28 Nov 2019 17:16:18 +0100 Subject: [PATCH 017/129] [SEARCH-1960] applied patch for including content value from cachedDoc only if the contentProperty has a fieldname in solr. --- .../org/alfresco/solr/SolrInformationServer.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index b1f8e871a..e545e35b4 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -2367,13 +2367,19 @@ public class SolrInformationServer implements InformationServer String transformationStatusFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS); - newDoc.addField(transformationStatusFieldName, cachedDoc.getFieldValue(transformationStatusFieldName)); + if (transformationStatusFieldName != null){ + newDoc.addField(transformationStatusFieldName, cachedDoc.getFieldValue(transformationStatusFieldName)); + } String transformationExceptionFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_EXCEPTION); - newDoc.addField(transformationExceptionFieldName, cachedDoc.getFieldValue(transformationExceptionFieldName)); - String transformationTimeFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, + if (transformationExceptionFieldName != null){ + newDoc.addField(transformationExceptionFieldName, cachedDoc.getFieldValue(transformationExceptionFieldName)); + } + String transformationTimeFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME); - newDoc.addField(transformationTimeFieldName, cachedDoc.getFieldValue(transformationTimeFieldName)); + if (transformationTimeFieldName != null){ + newDoc.addField(transformationTimeFieldName, cachedDoc.getFieldValue(transformationTimeFieldName)); + } // Gets the new content docid and compares to that of the cachedDoc to mark the content as clean/dirty String fldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, From 38e7c75030e9797c7143d87e9b9dd09d450ec907 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 28 Nov 2019 17:17:19 +0100 Subject: [PATCH 018/129] [SEARCH-1960] added integration test --- .../ContentPropertyValueTrackerIT.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java new file mode 100644 index 000000000..52fade531 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2005-2019 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import org.alfresco.model.ContentModel; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.ContentPropertyValue; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.StringPropertyValue; +import org.alfresco.solr.client.Transaction; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Locale; + +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.list; + +/** + * @author Elia Porciani + */ +@SolrTestCaseJ4.SuppressSSL +@SolrTestCaseJ4.SuppressObjectReleaseTracker (bugUrl = "RAMDirectory") +@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) +public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT +{ + + final private String authorField = "text@s__lt@{http://www.alfresco.org/model/content/1.0}author"; + + @BeforeClass + private static void initData() throws Throwable + { + initSolrServers(1, "ContentPropertyValueIndexIT", null); + } + + @AfterClass + private static void destroyData() throws Throwable + { + dismissSolrServers(); + } + + /** + * + * MNT-21076 (SEARCH-1906) + * + * This test aims to test if a document with more than one ContentPropertyValue, + * is removed from the index after any update. + * + * The ContentPropertyValue must not be indexed. + * @throws Exception + */ + @Test + public void testDataWithMultipleContentValueIsIndexedAfterUpdateTest() throws Exception + { + putHandleDefaults(); + AclChangeSet aclChangeSet = getAclChangeSet(1, 1); + Acl acl = getAcl(aclChangeSet); + AclReaders aclReaders = getAclReaders(aclChangeSet, acl, list("joel"), list("phil"), null); + indexAclChangeSet(aclChangeSet, + list(acl), + list(aclReaders)); + + Transaction txn = getTransaction(0, 1); + Node fileNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", null, false); + String author = "Mario"; + + // Here we set the PROP_TITLE as a ContentPropertyValye. + // The value is not indexed because the the related indexedField is not found. + fileMetaData.getProperties() + .put(ContentModel.PROP_TITLE, new ContentPropertyValue(Locale.CANADA, 100, "UTF8", "txt", 10l)); + fileMetaData.getProperties().put(ContentModel.PROP_AUTHOR, new StringPropertyValue(author)); + indexTransaction(txn, + list(fileNode), + list(fileMetaData)); + + // Check the document is correctly indexed/ + waitForDocCount(new TermQuery(new Term(authorField, author)), 1, MAX_WAIT_TIME); + Transaction txn1 = getTransaction(0, 1); + String authorAfterUpdate = "Luigi"; + fileMetaData.getProperties().put(ContentModel.PROP_AUTHOR, new StringPropertyValue(authorAfterUpdate)); + indexTransaction(txn1, + list(fileNode), + list(fileMetaData)); + + // Check the document is still indexed after the update. + waitForDocCount(new TermQuery(new Term(authorField, authorAfterUpdate)), 1, MAX_WAIT_TIME); + } +} \ No newline at end of file From 61bc1dd9a1241eaec3de28bd61af11906e4854e2 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 28 Nov 2019 17:21:42 +0100 Subject: [PATCH 019/129] [SEARCH-1960] typo and description fixes --- .../solr/tracker/ContentPropertyValueTrackerIT.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java index 52fade531..5696d449a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java @@ -52,7 +52,6 @@ import static org.alfresco.solr.AlfrescoSolrUtils.list; * @author Elia Porciani */ @SolrTestCaseJ4.SuppressSSL -@SolrTestCaseJ4.SuppressObjectReleaseTracker (bugUrl = "RAMDirectory") @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT { @@ -62,7 +61,7 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT @BeforeClass private static void initData() throws Throwable { - initSolrServers(1, "ContentPropertyValueIndexIT", null); + initSolrServers(1, "ContentPropertyValueTrackerIT", null); } @AfterClass @@ -75,10 +74,10 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT * * MNT-21076 (SEARCH-1906) * - * This test aims to test if a document with more than one ContentPropertyValue, - * is removed from the index after any update. + * This test aims to test that a document with more than one ContentPropertyValue + * is not removed from the index after any update. * - * The ContentPropertyValue must not be indexed. + * The second ContentPropertyValue must not be indexed. * @throws Exception */ @Test From b7815142dcd787f908421f8de19a98d3f797b6be Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2019 22:15:19 +0000 Subject: [PATCH 020/129] Bump alfresco-data-model from 8.55 to 8.57 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.55 to 8.57. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.55...8.57) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 811e1893e..8f526f1c2 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.55 + 8.57 2.10.1 From 37291f05a62bd414b3f53df21ffe204d1a9afd15 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 29 Nov 2019 08:45:53 +0000 Subject: [PATCH 021/129] Remove spring-jcl from license notice. --- search-services/packaging/src/main/resources/licenses/notice.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 6ebd88fb5..423541459 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -64,7 +64,6 @@ spring-context-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ spring-context-support-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ spring-core-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ spring-expression-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-jcl-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ spring-jdbc-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ spring-orm-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ spring-tx-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ From 0bf031ffe47b46eac5efffe23b1465d1ff84ae95 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 29 Nov 2019 14:36:32 +0000 Subject: [PATCH 022/129] SEARCH-1989 Refactor SolrSearchByPropertyTests. Convert the brittle xml-based result-count tests into 'normal' result set tests. Remove all tests for string comparison using less than/greater than. This is not supported according to the CMIS spec. See "2.1.14.2.4.1 Comparisons permitted in the WHERE clause" here: http://docs.oasis-open.org/cmis/CMIS/v1.1/csprd01/CMIS-v1.1-csprd01.html#x1-10500014 Nb. String ordering (in an ORDER BY clause) is repository specific - i.e. it's not specified in the CMIS spec. Assume that our current ordering is how we want it to be (e.g. '1' comes before '.'). --- e2e-test/pom.xml | 2 +- .../cmis/SolrSearchByPropertyTests.java | 443 +++++++++++++++++- .../resources/testdata/search-by-property.xml | 188 -------- 3 files changed, 422 insertions(+), 211 deletions(-) delete mode 100644 e2e-test/src/test/resources/testdata/search-by-property.xml diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 195de617a..8e716d9b1 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -12,7 +12,7 @@ Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share 1.15 - 1.6 + 1.9 3.0.14 3.2.0 src/test/resources/SearchSuite.xml diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java index 40ea49483..d9f7638bd 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java @@ -1,5 +1,8 @@ package org.alfresco.test.search.functional.searchServices.cmis; +import java.util.List; +import java.util.Set; + import org.alfresco.utility.Utility; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.data.provider.XMLDataConfig; @@ -15,7 +18,7 @@ public class SolrSearchByPropertyTests extends AbstractCmisE2ETest private FolderModel guestf, tesf, restf, testtttf, testf, testf1, testf2, testf3, testf4; private FileModel guestc, restc, tesc, testtttc, testc, testc1, testc2, testc3; - @BeforeClass(alwaysRun = true) + @BeforeClass (alwaysRun = true) public void dataPreparation() throws Exception { dataContent.usingAdmin().deployContentModel("model/tas-model.xml"); @@ -46,68 +49,464 @@ public class SolrSearchByPropertyTests extends AbstractCmisE2ETest // Sites dataContent.usingUser(testUser).usingSite(testSite).createCustomContent(guestf, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "guestf text") - .addProperty("tas:IntPropertyF", 222)); + .addProperty("tas:IntPropertyF", 222)); dataContent.usingUser(testUser).usingSite(testSite).createCustomContent(tesf, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "tesf text") - .addProperty("tas:IntPropertyF", 224)); + .addProperty("tas:IntPropertyF", 224)); // Sites >> Folders dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(restf, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "restf text") - .addProperty("tas:IntPropertyF", 223)); + .addProperty("tas:IntPropertyF", 223)); dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(testtttf, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "testtttf text") - .addProperty("tas:IntPropertyF", 225)); + .addProperty("tas:IntPropertyF", 225)); dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(testf, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "testf text") - .addProperty("tas:IntPropertyF", 226)); + .addProperty("tas:IntPropertyF", 226)); dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(testf1, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "testf1 text") - .addProperty("tas:IntPropertyF", 2221)); + .addProperty("tas:IntPropertyF", 2221)); dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(testf2, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "testf2 text") - .addProperty("tas:IntPropertyF", 2222)); + .addProperty("tas:IntPropertyF", 2222)); dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(testf3, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "testf3 text") - .addProperty("tas:IntPropertyF", 2223)); + .addProperty("tas:IntPropertyF", 2223)); dataContent.usingUser(testUser).usingResource(guestf).createCustomContent(testf4, "F:tas:folder", new CustomObjectTypeProperties().addProperty("tas:TextPropertyF", "testf4 text") - .addProperty("tas:IntPropertyF", 2224)); + .addProperty("tas:IntPropertyF", 2224)); // Sites >> Files dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(guestc, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "guestc text") - .addProperty("tas:IntPropertyC", 222)); + .addProperty("tas:IntPropertyC", 222)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(restc, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "restc text") - .addProperty("tas:IntPropertyC", 223)); + .addProperty("tas:IntPropertyC", 223)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(tesc, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "tesc text") - .addProperty("tas:IntPropertyC", 224)); + .addProperty("tas:IntPropertyC", 224)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(testtttc, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "testtttc text") - .addProperty("tas:IntPropertyC", 225)); + .addProperty("tas:IntPropertyC", 225)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(testc, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "testc text") - .addProperty("tas:IntPropertyC", 226)); + .addProperty("tas:IntPropertyC", 226)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(testc1, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "testc1 text") - .addProperty("tas:IntPropertyC", 2221)); + .addProperty("tas:IntPropertyC", 2221)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(testc2, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "testc2 text") - .addProperty("tas:IntPropertyC", 2222)); + .addProperty("tas:IntPropertyC", 2222)); dataContent.usingUser(testUser).usingResource(tesf).createCustomContent(testc3, "D:tas:document", new CustomObjectTypeProperties().addProperty("tas:TextPropertyC", "testc3 text") - .addProperty("tas:IntPropertyC", 2223)); + .addProperty("tas:IntPropertyC", 2223)); // wait for solr index Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") - @XMLDataConfig(file = "src/test/resources/testdata/search-by-property.xml") - public void executeSearchByProperty(QueryModel query) throws Exception + @Test + public void testFileNameEquality() { - cmisApi.authenticateUser(testUser).withQuery(query.getValue()).assertResultsCount().equals(query.getResults()); + String query = "SELECT * FROM tas:document where cmis:name = 'testc.txt'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt")); + } + + @Test + public void testFileNameInequality() + { + String query = "SELECT * FROM tas:document where cmis:name <> 'testc.txt'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "restc.txt", "tesc.txt", "testtttc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileNameIn() + { + String query = "SELECT * FROM tas:document where cmis:name IN('testc.txt', 'guestc.txt', 'restc.txt')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt", "guestc.txt", "restc.txt")); + } + + @Test + public void testFileNameNotIn() + { + // Nb. "gustc" is missing an "e". + String query = "SELECT * FROM tas:document where cmis:name NOT IN('testc.txt', 'gustc.txt', 'restc.txt')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "tesc.txt", "testtttc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileNameLike() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE '%testc%'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileNameLikeExact() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE 'testc.txt'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt")); + } + + @Test + public void testFileNamePrefixSuffix() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE 't%tc.txt'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt", "testtttc.txt")); + } + + @Test + public void testFileNameUnderscore() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE 't__tc.txt'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt")); + } + + @Test + public void testFolderNameEquality() + { + String query = "SELECT * FROM tas:folder where cmis:name = 'testf'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testf")); + } + + @Test + public void testFolderNameInequality() + { + String query = "SELECT * FROM tas:folder where cmis:name <> 'testf'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "restf", "testtttf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderNameIn() + { + String query = "SELECT * FROM tas:folder where cmis:name IN('testf', 'guestf', 'restf')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testf", "guestf", "restf")); + } + + @Test + public void testFolderNameNotIn() + { + // Nb. "gustc" is missing an "e". + String query = "SELECT * FROM tas:folder where cmis:name NOT IN('testf', 'gustf', 'restf')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "testtttf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderNameLike() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE '%testf%'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderNameLikeExact() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE 'testf'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testf")); + } + + @Test + public void testFolderNamePrefixSuffix() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE 't%tf'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testtttf", "testf")); + } + + @Test + public void testFolderNameUnderscore() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE 't__tf'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testf")); + } + + @Test + public void testFileNameOrderAsc() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE '%testc%' ORDER BY cmis:name ASC"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningOrderedValues("cmis:name", + List.of("testc1.txt", "testc2.txt", "testc3.txt", "testc.txt")); + } + + @Test + public void testFileNameOrderDesc() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE '%testc%' ORDER BY cmis:name DESC"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningOrderedValues("cmis:name", + List.of("testc.txt", "testc3.txt", "testc2.txt", "testc1.txt")); + } + + @Test + public void testFileOrderNameDescDateAsc() + { + String query = "SELECT * FROM tas:document where cmis:name LIKE '%testc%' ORDER BY cmis:name DESC, cmis:creationDate ASC"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningOrderedValues("cmis:name", + List.of("testc.txt", "testc3.txt", "testc2.txt", "testc1.txt")); + } + + @Test + public void testFolderNameOrderAsc() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE '%testf%' ORDER BY cmis:name ASC"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningOrderedValues("cmis:name", + List.of("testf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderNameOrderDesc() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE '%testf%' ORDER BY cmis:name DESC"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningOrderedValues("cmis:name", + List.of("testf4", "testf3", "testf2", "testf1", "testf")); + } + + @Test + public void testFolderOrderNameDescDateAsc() + { + String query = "SELECT * FROM tas:folder where cmis:name LIKE '%testf%' ORDER BY cmis:name DESC, cmis:creationDate ASC"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningOrderedValues("cmis:name", + List.of("testf4", "testf3", "testf2", "testf1", "testf")); + } + + @Test + public void testFileCustomPropertyEquality() + { + String query = "SELECT * FROM tas:document where tas:TextPropertyC = 'restc text'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("restc.txt")); + } + + @Test + public void testFileCustomPropertyInequality() + { + String query = "SELECT * FROM tas:document where tas:TextPropertyC <> 'testc1 text'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "restc.txt", "tesc.txt", "testtttc.txt", "testc.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileCustomPropertyIn() + { + String query = "SELECT * FROM tas:document where tas:TextPropertyC IN('restc text', 'testc2 text', 'testc3 text')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("restc.txt", "testc2.txt", "testc3.txt")); + } + + + @Test + public void testFileCustomPropertyNotIn() + { + String query = "SELECT * FROM tas:document where tas:TextPropertyC NOT IN('restc text', 'testc2 text', 'testc3 text')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "tesc.txt", "testtttc.txt", "testc.txt", "testc1.txt")); + } + + @Test + public void testFileCustomPropertyLike() + { + String query = "SELECT * FROM tas:document where tas:TextPropertyC LIKE '%restc%'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("restc.txt")); + } + + @Test + public void testFileCustomPropertyUnderscore() + { + String query = "SELECT * FROM tas:document where tas:TextPropertyC LIKE 't__tc text'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testc.txt")); + } + + @Test + public void testFolderCustomPropertyEquality() + { + String query = "SELECT * FROM tas:folder where tas:TextPropertyF = 'restf text'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("restf")); + } + + @Test + public void testFolderCustomPropertyInequality() + { + String query = "SELECT * FROM tas:folder where tas:TextPropertyF <> 'testf1 text'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "restf", "testtttf", "testf", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderCustomPropertyIn() + { + String query = "SELECT * FROM tas:folder where tas:TextPropertyF IN('restf text', 'testf2 text', 'testf3 text')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("restf", "testf2", "testf3")); + } + + @Test + public void testFolderCustomPropertyNotIn() + { + String query = "SELECT * FROM tas:folder where tas:TextPropertyF NOT IN('restf text', 'testf2 text', 'testf3 text')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "testtttf", "testf", "testf1", "testf4")); + } + + @Test + public void testFolderCustomPropertyLike() + { + String query = "SELECT * FROM tas:folder where tas:TextPropertyF LIKE '%restf%'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("restf")); + } + + @Test + public void testFolderCustomPropertyUnderscore() + { + String query = "SELECT * FROM tas:folder where tas:TextPropertyF LIKE 't__tf text'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testf")); + } + + @Test + public void testFileIntEquality() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC = '222'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt")); + } + + @Test + public void testFileIntInequality() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC <> '223'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "tesc.txt", "testtttc.txt", "testc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileIntLessThen() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC < '223'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt")); + } + + @Test + public void testFileIntLessThanOrEqual() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC <= '224'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "restc.txt", "tesc.txt")); + } + + @Test + public void testFileIntGreaterThanOrEqual() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC >= '224'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("tesc.txt", "testtttc.txt", "testc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileIntGreaterThan() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC > '224'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testtttc.txt", "testc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFileIntIn() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC IN('222', '223', '224', '225')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestc.txt", "restc.txt", "tesc.txt", "testtttc.txt")); + } + + @Test + public void testFileIntNotIn() + { + String query = "SELECT * FROM tas:document where tas:IntPropertyC NOT IN('222', '223', '224')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testtttc.txt", "testc.txt", "testc1.txt", "testc2.txt", "testc3.txt")); + } + + @Test + public void testFolderIntEquality() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF = '222'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf")); + } + + @Test + public void testFolderIntInequality() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF <> '223'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "testtttf", "testf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderIntLessThan() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF < '223'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf")); + } + + @Test + public void testFolderIntLessThanOrEqual() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF <= '224'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "restf")); + } + + @Test + public void testFolderIntGreaterThanOrEqual() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF >= '224'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("tesf", "testtttf", "testf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderIntGreaterThan() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF > '224'"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testtttf", "testf", "testf1", "testf2", "testf3", "testf4")); + } + + @Test + public void testFolderIntIn() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF IN('222', '223', '224', '225')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("guestf", "tesf", "restf", "testtttf")); + } + + @Test + public void testFolderIntNotIn() + { + String query = "SELECT * FROM tas:folder where tas:IntPropertyF NOT IN('222', '223', '224')"; + cmisApi.authenticateUser(testUser).withQuery(query).assertValues().isReturningValues("cmis:name", + Set.of("testtttf", "testf", "testf1", "testf2", "testf3", "testf4")); } } diff --git a/e2e-test/src/test/resources/testdata/search-by-property.xml b/e2e-test/src/test/resources/testdata/search-by-property.xml deleted file mode 100644 index 0b845ec67..000000000 --- a/e2e-test/src/test/resources/testdata/search-by-property.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From f4ea83b9bbb23e6dff4c0c0a1fb6b4b4c466fe21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2019 22:12:14 +0000 Subject: [PATCH 023/129] Bump utility from 3.0.14 to 3.0.15 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.14 to 3.0.15. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.14...utility-3.0.15) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 195de617a..f170f842d 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 1.15 1.6 - 3.0.14 + 3.0.15 3.2.0 src/test/resources/SearchSuite.xml From 46f2c2bfac4493af27343f93622d2dff427b1efb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2019 22:12:14 +0000 Subject: [PATCH 024/129] Bump mockito-core from 3.1.0 to 3.2.0 in /search-services Bumps [mockito-core](https://github.com/mockito/mockito) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.1.0...v3.2.0) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 0be13e026..3a1a3401d 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -158,7 +158,7 @@ org.mockito mockito-core - 3.1.0 + 3.2.0 test diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 8f526f1c2..9438343fb 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -67,7 +67,7 @@ org.mockito mockito-core - 3.1.0 + 3.2.0 test From 919c3290738de5cb0b36a6bc62c6046548ccf06f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2019 22:12:36 +0000 Subject: [PATCH 025/129] Bump restapi from 1.15 to 1.17 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.15 to 1.17. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.15...v1.17) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 195de617a..a1c72520d 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.15 + 1.17 1.6 3.0.14 3.2.0 From 1e1502b803f4cb6a01220bd1b04aca14f2f7809b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2019 10:26:11 +0000 Subject: [PATCH 026/129] Bump cmis from 1.6 to 1.10 in /e2e-test Bumps [cmis](https://github.com/Alfresco/alfresco-tas-cmis) from 1.6 to 1.10. - [Release notes](https://github.com/Alfresco/alfresco-tas-cmis/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-cmis/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-cmis/compare/v1.6...v1.10) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index b109566a7..d011bd6ce 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -12,7 +12,7 @@ Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share 1.17 - 1.6 + 1.10 3.0.15 3.2.0 src/test/resources/SearchSuite.xml From 7637f9ebe9d90c7f7abe2d51e942da2d41ab3779 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2019 13:14:04 +0000 Subject: [PATCH 027/129] Bump restapi from 1.17 to 1.18 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.17 to 1.18. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.17...v1.18) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index d011bd6ce..187ee9212 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.17 + 1.18 1.10 3.0.15 3.2.0 From 10c4eff30cc8985fd6576225f762f7967f9766c7 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Wed, 4 Dec 2019 13:26:53 +0100 Subject: [PATCH 028/129] Bump Docker Image release numbers to ACS 6.2 --- .../generators/app/templates/6.2/.env | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env index b39fd3b1e..352f5b342 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env @@ -1,15 +1,15 @@ ALFRESCO_TAG=latest -SHARE_TAG=6.1.0-RC3 -POSTGRES_TAG=10.9 -TRANSFORM_ROUTER_TAG=1.1.0-EA2 -PDF_RENDERER_TAG=2.1.0-EA4 -IMAGE_MAGICK_TAG=2.1.0-EA4 -LIBREOFFICE_TAG=2.1.0-EA4 -TIKA_TAG=2.1.0-EA4 -TRANSFORM_MISC_TAG=2.1.0-EA4 +SHARE_TAG=6.2.0 +POSTGRES_TAG=11.4 +TRANSFORM_ROUTER_TAG=1.1.0 +PDF_RENDERER_TAG=2.1.0 +IMAGE_MAGICK_TAG=2.1.0 +LIBREOFFICE_TAG=2.1.0 +TIKA_TAG=2.1.0 +TRANSFORM_MISC_TAG=2.1.0 SHARED_FILE_STORE_TAG=0.5.3 ACTIVE_MQ_TAG=5.15.8 -DIGITAL_WORKSPACE_TAG=1.1.0 +DIGITAL_WORKSPACE_TAG=1.3.0 ACS_NGINX_TAG=3.0.1 ACS_COMMUNITY_NGINX_TAG=1.0.0 SEARCH_TAG=latest From 10ad96fd2af51e551a952437d464fdf2fdd847aa Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 4 Dec 2019 15:37:28 +0100 Subject: [PATCH 029/129] [SEARCH-1960] removed unnecessary Exception from test method throw lists --- .../test/java/org/alfresco/solr/AdminHandlerDistributedIT.java | 2 +- .../java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java | 2 +- .../test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java | 2 +- .../src/test/java/org/alfresco/solr/TemplatesDistributedIT.java | 2 +- .../solr/highlight/AlfrescoHighligherDistributedIT.java | 2 +- .../alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java | 2 +- .../solr/query/DistributedAlfrescoSolrSpellcheckerIT.java | 2 +- .../alfresco/solr/tracker/ContentPropertyValueTrackerIT.java | 2 +- .../solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java | 2 +- .../alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java | 2 +- .../alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java | 2 +- .../solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java | 2 +- .../tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java | 2 +- .../tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java | 2 +- .../solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java | 2 +- .../DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java | 2 +- .../DistributedExplicitShardIdWithStaticPropertyRouterIT.java | 2 +- .../solr/tracker/DistributedExplicitShardRoutingTrackerIT.java | 2 +- .../tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java | 2 +- .../org/alfresco/solr/transformer/CachedDocTransformerIT.java | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java index 2a9fe6aca..613f9fd63 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AdminHandlerDistributedIT.java @@ -56,7 +56,7 @@ public class AdminHandlerDistributedIT extends AbstractAlfrescoDistributedIT } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java index feb54a648..009290dbc 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateUpdateDistributedIT.java @@ -63,7 +63,7 @@ public class CoresCreateUpdateDistributedIT extends AbstractAlfrescoDistributedI } @After - private void destroyData() throws Throwable + private void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java index 82d128394..432b63c79 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CoresCreateViaPropertyIT.java @@ -53,7 +53,7 @@ public class CoresCreateViaPropertyIT extends AbstractAlfrescoDistributedIT } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); System.clearProperty(AlfrescoCoreAdminHandler.ALFRESCO_DEFAULTS); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java index b7db5c77a..13ce20329 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/TemplatesDistributedIT.java @@ -52,7 +52,7 @@ public class TemplatesDistributedIT extends AbstractAlfrescoDistributedIT } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java index 4eb9cae4c..93c3bdab5 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/highlight/AlfrescoHighligherDistributedIT.java @@ -46,7 +46,7 @@ public class AlfrescoHighligherDistributedIT extends AbstractAlfrescoDistributed } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java index 13f28d874..925d7362e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrFacetingIT.java @@ -48,7 +48,7 @@ import static org.hamcrest.core.Is.is; } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java index 1561564ed..23861134b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/DistributedAlfrescoSolrSpellcheckerIT.java @@ -42,7 +42,7 @@ public class DistributedAlfrescoSolrSpellcheckerIT extends AbstractAlfrescoDistr } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java index 5696d449a..2f9c2dca3 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java @@ -65,7 +65,7 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java index 8c268e01a..709831de8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAclIdAlfrescoSolrTrackerIT.java @@ -67,7 +67,7 @@ public class DistributedAclIdAlfrescoSolrTrackerIT extends AbstractAlfrescoDistr } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java index c0d0c14ec..06abaf5cc 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrJsonIT.java @@ -59,7 +59,7 @@ public class DistributedAlfrescoSolrJsonIT extends AbstractAlfrescoDistributedIT } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java index 8b0f610f9..bc31c87ad 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerIT.java @@ -66,7 +66,7 @@ public class DistributedAlfrescoSolrTrackerIT extends AbstractAlfrescoDistribute } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java index 01290fcc7..764dd1237 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java @@ -64,7 +64,7 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java index f2dc207c6..4464ce079 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateQuarterAlfrescoSolrTrackerIT.java @@ -43,7 +43,7 @@ public class DistributedDateQuarterAlfrescoSolrTrackerIT extends DistributedDate } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java index ff488c138..0407cfdfe 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateSplitYearAlfrescoSolrTrackerIT.java @@ -42,7 +42,7 @@ public class DistributedDateSplitYearAlfrescoSolrTrackerIT extends DistributedDa } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java index f683c0ce1..ddf5117fc 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDbidRangeAlfrescoSolrTrackerIT.java @@ -62,7 +62,7 @@ public class DistributedDbidRangeAlfrescoSolrTrackerIT extends AbstractAlfrescoD } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java index 07c07f0b7..2be51b0da 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java @@ -65,7 +65,7 @@ public class DistributedExpandDbidRangeAlfrescoSolrTrackerIT extends AbstractAlf } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java index 7ccf1792e..7b6bb47a8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardIdWithStaticPropertyRouterIT.java @@ -66,7 +66,7 @@ public class DistributedExplicitShardIdWithStaticPropertyRouterIT extends Abstra } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java index bd7c5f325..c69968d76 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExplicitShardRoutingTrackerIT.java @@ -71,7 +71,7 @@ public class DistributedExplicitShardRoutingTrackerIT extends AbstractAlfrescoDi } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java index 1e7c8b8f4..ca95e72c9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedPropertyBasedAlfrescoSolrTrackerIT.java @@ -82,7 +82,7 @@ public class DistributedPropertyBasedAlfrescoSolrTrackerIT extends AbstractAlfre } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java index 4d0314e3c..a9c6ae65b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java @@ -70,7 +70,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT } @AfterClass - private static void destroyData() throws Throwable + private static void destroyData() { dismissSolrServers(); } From 9979872ab150c6b11222b4f3842b355464fb8422 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 4 Dec 2019 16:54:19 +0100 Subject: [PATCH 030/129] [SEARCH-1960] Added comments in test --- .../tracker/ContentPropertyValueTrackerIT.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java index 2f9c2dca3..55263b4ac 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentPropertyValueTrackerIT.java @@ -74,8 +74,9 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT * * MNT-21076 (SEARCH-1906) * - * This test aims to test that a document with more than one ContentPropertyValue - * is not removed from the index after any update. + * This test aims to test that a document with more than one (not indexed) ContentPropertyValue + * is not removed from the index after any update. The fix prevents an NPE during an update because the + * not indexed ContentPropertyValue is searched and not found in the cached document. * * The second ContentPropertyValue must not be indexed. * @throws Exception @@ -86,6 +87,8 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT putHandleDefaults(); AclChangeSet aclChangeSet = getAclChangeSet(1, 1); Acl acl = getAcl(aclChangeSet); + + // Arbitrary acl data. AclReaders aclReaders = getAclReaders(aclChangeSet, acl, list("joel"), list("phil"), null); indexAclChangeSet(aclChangeSet, list(acl), @@ -96,8 +99,8 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", null, false); String author = "Mario"; - // Here we set the PROP_TITLE as a ContentPropertyValye. - // The value is not indexed because the the related indexedField is not found. + // Here we set the PROP_TITLE as a ContentPropertyValue. + // The value is not indexed because the related indexedField is not found (it does not exist). fileMetaData.getProperties() .put(ContentModel.PROP_TITLE, new ContentPropertyValue(Locale.CANADA, 100, "UTF8", "txt", 10l)); fileMetaData.getProperties().put(ContentModel.PROP_AUTHOR, new StringPropertyValue(author)); @@ -105,8 +108,10 @@ public class ContentPropertyValueTrackerIT extends AbstractAlfrescoDistributedIT list(fileNode), list(fileMetaData)); - // Check the document is correctly indexed/ + // Check the document is correctly indexed waitForDocCount(new TermQuery(new Term(authorField, author)), 1, MAX_WAIT_TIME); + + // Update the author. Transaction txn1 = getTransaction(0, 1); String authorAfterUpdate = "Luigi"; fileMetaData.getProperties().put(ContentModel.PROP_AUTHOR, new StringPropertyValue(authorAfterUpdate)); From 588b83ba95c1421587ab98ce66df5533d37c2708 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2019 22:14:04 +0000 Subject: [PATCH 031/129] Bump alfresco-data-model from 8.57 to 8.60 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.57 to 8.60. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.57...8.60) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 9438343fb..eb8c5dd03 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.57 + 8.60 2.10.1 From b8f40b0c6ca49d6be84cf426de918720bea337ff Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2019 22:14:16 +0000 Subject: [PATCH 032/129] Bump utility from 3.0.15 to 3.0.16 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.15 to 3.0.16. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.15...utility-3.0.16) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 187ee9212..80f10d360 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 1.18 1.10 - 3.0.15 + 3.0.16 3.2.0 src/test/resources/SearchSuite.xml From d9c1a153fb59daa1c5d5489c0e435ac65816bdce Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 6 Dec 2019 15:10:10 +0100 Subject: [PATCH 033/129] [SEARCH-1994] modified path computation in order to work correctly on windows. --- .../solr/handler/AlfrescoIndexFetcher.java | 141 +++++++++--------- 1 file changed, 72 insertions(+), 69 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index ee1666778..3723b22e7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -34,39 +34,9 @@ */ package org.alfresco.solr.handler; -import static java.util.List.of; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES; -import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum; -import static org.apache.solr.common.params.CommonParams.JAVABIN; -import static org.apache.solr.common.params.CommonParams.NAME; - import com.google.common.base.Strings; import com.google.common.collect.Lists; +import jdk.internal.jline.internal.Log; import org.alfresco.solr.content.SolrContentStore; import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; @@ -152,6 +122,37 @@ import java.util.zip.Adler32; import java.util.zip.Checksum; import java.util.zip.InflaterInputStream; +import static java.util.List.of; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.ALIAS; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CHECKSUM; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_DETAILS; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_GET_FILE_LIST; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CMD_INDEX_VERSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMMAND; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.COMPRESSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONF_FILE_SHORT; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_FILE_LIST; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.CONTENT_STORE_VERSION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.EXTERNAL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FILE_STREAM; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.FileInfo; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.GENERATION; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.INTERNAL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.MASTER_URL; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.NO_INDEX_REPLICATION_REQUIRED; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.OFFSET; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.SIZE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILE; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.TLOG_FILES; +import static org.alfresco.solr.handler.AlfrescoReplicationHandler.getCheckSum; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.NAME; + /** *

Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the * master.

@@ -776,30 +777,37 @@ class AlfrescoIndexFetcher } } - if (contentStoreReplicationNeeded) - { + try { - if (contentStoreFilesToDownload != null) + if (contentStoreReplicationNeeded) { - bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation()); + + if (contentStoreFilesToDownload != null) + { + bytesDownloaded += downloadContentStoreFiles(contentStore.getRootLocation()); + } + + if (contentStoreFilesToDelete != null) + { + deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete); + } + + if (fullContentStoreReplication) + { + cleanUpContentStore(contentStore.getRootLocation()); + } + + contentStore.setLastCommittedVersion(masterContentStoreVersion); + LOG.info("content store has been updated to version: {}", masterContentStoreVersion); } - if (contentStoreFilesToDelete != null) - { - deleteContentStoreFiles(contentStore.getRootLocation(), contentStoreFilesToDelete); - } - - if (fullContentStoreReplication) - { - cleanUpContentStore(contentStore.getRootLocation()); - } - - contentStore.setLastCommittedVersion(masterContentStoreVersion); + } catch (Exception e) { + LOG.error("impossible to complete content store replication {}", e); } final long timeTakenSeconds = getReplicationTimeElapsed(); final Long bytesDownloadedPerSecond = (timeTakenSeconds != 0 ? bytesDownloaded / timeTakenSeconds : - null); + null); LOG.info("Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec)", isFullCopyNeeded, bytesDownloaded, timeTakenSeconds, bytesDownloadedPerSecond); @@ -1683,37 +1691,32 @@ class AlfrescoIndexFetcher * @param contentStorePath * @throws IOException */ - private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws IOException + private void copyTmpContentStoreToContentStore(File tmpContentStoreDir, String contentStorePath) throws Exception { String tmpContentStorePath = tmpContentStoreDir.getPath(); - try - { - Files.walk(tmpContentStoreDir.toPath()).forEach(p -> { - File tmpFile = new File(p.toUri()); - if (!tmpFile.isDirectory()) + Files.walk(tmpContentStoreDir.toPath()).forEach(p -> { + File tmpFile = new File(p.toUri()); + if (!tmpFile.isDirectory()) + { + File csFile = new File(p.toString().replace(tmpContentStorePath, contentStorePath)); + try { - File csFile = new File(p.toString().replaceFirst(tmpContentStorePath, contentStorePath)); - try - { - Files.createDirectories(Paths.get(csFile.getParent())); - tmpFile.renameTo(csFile); - } - catch (IOException e) - { - LOG.error("impossible to copy {}", csFile.toString()); + Files.createDirectories(Paths.get(csFile.getParent())); + if (!tmpFile.renameTo(csFile)){ + throw new RuntimeException("Failed while moving content store file to " + csFile.getAbsolutePath()); } } - }); - } - catch (IOException e) - { - LOG.error("impossible tmp content store"); - throw e; - } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + }); } + /** * Deletes the files in filesToDelete list from contentStore * @param contentStorePath From b6f038c09c3af29223bc4d5ab686f7b974fe7632 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 6 Dec 2019 16:35:51 +0100 Subject: [PATCH 034/129] [SEARCH-1994] fix compilation error --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 1 - 1 file changed, 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 3723b22e7..477a38dec 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -36,7 +36,6 @@ package org.alfresco.solr.handler; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import jdk.internal.jline.internal.Log; import org.alfresco.solr.content.SolrContentStore; import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; From e1fa77e4bfa594bfb49291ee7bbaa09f514124f6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2019 22:13:09 +0000 Subject: [PATCH 035/129] Bump restapi from 1.18 to 1.20 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.18 to 1.20. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.18...v1.20) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 187ee9212..f8a05c853 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.18 + 1.20 1.10 3.0.15 3.2.0 From ab301c0d9f4aed540fd5e729f89a2536cd70a10f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2019 22:13:48 +0000 Subject: [PATCH 036/129] Bump cmis from 1.10 to 1.11 in /e2e-test Bumps [cmis](https://github.com/Alfresco/alfresco-tas-cmis) from 1.10 to 1.11. - [Release notes](https://github.com/Alfresco/alfresco-tas-cmis/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-cmis/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-cmis/compare/v1.10...v1.11) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 187ee9212..5293fdaf7 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -12,7 +12,7 @@ Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share 1.18 - 1.10 + 1.11 3.0.15 3.2.0 src/test/resources/SearchSuite.xml From 57c0fca88cc6cb4e251660032e323c0e79a9af7b Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 9 Dec 2019 08:38:20 +0000 Subject: [PATCH 037/129] SEARCH-1998 Upgrade Java base image. --- search-services/packaging/src/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/Dockerfile b/search-services/packaging/src/docker/Dockerfile index 852223ecf..ea60bca76 100644 --- a/search-services/packaging/src/docker/Dockerfile +++ b/search-services/packaging/src/docker/Dockerfile @@ -1,6 +1,6 @@ # Alfresco Search Services ${project.version} Docker Image -FROM alfresco/alfresco-base-java:11.0.1-openjdk-centos-7-6784d76a7b81 +FROM alfresco/alfresco-base-java:11.0.1-openjdk-centos-7-7a6031154417 LABEL creator="Gethin James" maintainer="Alfresco Search Services Team" ENV DIST_DIR /opt/alfresco-search-services From 8a5ec109b6b513ae7c1facf8e8b4afb58e466189 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 9 Dec 2019 09:43:05 +0000 Subject: [PATCH 038/129] Update license for Spring 5.2.2.RELEASE. --- .../src/main/resources/licenses/notice.txt | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 423541459..ce669d559 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -58,16 +58,16 @@ jackson-core-2.10.1.jar https://github.com/FasterXML/jackson jackson-annotations-2.10.1.jar https://github.com/FasterXML/jackson jackson-databind-2.10.1.jar https://github.com/FasterXML/jackson commons-httpclient-3.1-HTTPCLIENT-1265.jar http://jakarta.apache.org/commons/ -spring-aop-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-beans-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-context-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-context-support-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-core-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-expression-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-jdbc-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-orm-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-tx-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-web-5.1.8.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-aop-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-beans-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-context-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-context-support-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-core-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-expression-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-jdbc-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-orm-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-tx-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-web-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ xercesImpl-2.12.0-alfresco-patched-20191004.jar http://xerces.apache.org/xerces2-j guessencoding-1.4.jar http://docs.codehaus.org/display/GUESSENC/ xml-apis-1.4.01.jar https://github.com/FasterXML/jackson From 25111a31422e60518d46fbd13f427b36ba73b828 Mon Sep 17 00:00:00 2001 From: Meenal Bhave Date: Mon, 9 Dec 2019 12:34:37 +0000 Subject: [PATCH 039/129] Add helper methods testSearchQueryOrdered and testSearchQueryUnordered. Refactor the first test to use this. --- .../java/org/alfresco/search/TestGroup.java | 1 + .../functional/AbstractE2EFunctionalTest.java | 161 ++++++--- .../search/SearchAFTSInFieldTest.java | 326 ++++++++++++++++++ 3 files changed, 434 insertions(+), 54 deletions(-) create mode 100644 e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java diff --git a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java index a149ab6f7..fbb90a64e 100644 --- a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java +++ b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java @@ -38,6 +38,7 @@ public class TestGroup public static final String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above public static final String ACS_611n = "ACS_611n"; // Alfresco Content Services 6.1.1 or above public static final String ACS_62n = "ACS_62n"; // Alfresco Content Services 6.2 or above + public static final String ACS_63n = "ACS_63n"; // Alfresco Content Services 6.3 or above public static final String AGS_302 = "AGS_302"; // Alfresco governance Services 3.0.2 or above diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index 705fb848d..8073d003d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -6,6 +6,16 @@ */ package org.alfresco.test.search.functional; +import static java.util.Optional.ofNullable; + +import static lombok.AccessLevel.PROTECTED; +import static org.testng.Assert.assertEquals; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import lombok.Getter; import org.alfresco.cmis.CmisWrapper; import org.alfresco.dataprep.ContentService; import org.alfresco.dataprep.SiteService.Visibility; @@ -38,15 +48,10 @@ import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; -import lombok.Getter; - -import static java.util.Optional.ofNullable; -import static lombok.AccessLevel.PROTECTED; - /** * @author meenal bhave */ -@ContextConfiguration("classpath:alfresco-search-e2e-context.xml") +@ContextConfiguration ("classpath:alfresco-search-e2e-context.xml") public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests { /** The number of retries that a query will be tried before giving up. */ @@ -76,26 +81,26 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont protected CmisWrapper cmisApi; @Autowired - // @Getter(value = PROTECTED) + // @Getter(value = PROTECTED) protected DataUser dataUser; @Autowired - @Getter(value = PROTECTED) + @Getter (value = PROTECTED) private ContentService contentService; protected UserModel testUser, adminUserModel, testUser2; protected SiteModel testSite, testSite2; protected static String unique_searchString; - + protected static final String SEARCH_LANGUAGE_CMIS = "cmis"; - + protected enum SearchLanguage { CMIS, AFTS - } + } - @BeforeSuite(alwaysRun = true) + @BeforeSuite (alwaysRun = true) public void beforeSuite() throws Exception { super.springTestContextPrepareTestInstance(); @@ -104,7 +109,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont deployCustomModel("model/finance-model.xml"); } - @BeforeClass(alwaysRun = true) + @BeforeClass (alwaysRun = true) public void setup() { serverHealth.assertServerIsOnline(); @@ -209,7 +214,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont customModel.setNodeRef(modelInRepo.getId()); customModel.setNodeRef(customModel.getNodeRefWithoutVersion()); customModel.setCmisLocation(String.format("/Data Dictionary/Models/%s", fileName)); - LOGGER.info("Custom Model file: " + customModel.getCmisLocation()); + LOGGER.info("Custom Model file: " + customModel.getCmisLocation()); } else { @@ -239,9 +244,9 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Wait for Solr to finish indexing and search to return appropriate results - * - * @param userQuery Search Query - * @param contentToFind that's expected to be included / excluded from the results + * + * @param userQuery Search Query + * @param contentToFind that's expected to be included / excluded from the results * @param expectedInResults Whether we expect the content in the results or not. * @return true if search returns expected results, i.e. is given content is found or excluded from the results */ @@ -277,25 +282,25 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont return false; } - + /** * Method to check if the contentName is returned in the SearchResponse. * - * @param response the search response + * @param response the search response * @param contentName the text we are using as matching/verifying criteria. * @return true if if the item with the contentName text is returned in the SearchResponse. */ - public boolean isContentInSearchResponse(SearchResponse response, String contentName) + public boolean isContentInSearchResponse(SearchResponse response, String contentName) { return response.getEntries().stream() - .map(entry -> entry.getModel().getName()) - .anyMatch(name -> name.equalsIgnoreCase(contentName) || contentName.isBlank()); + .map(entry -> entry.getModel().getName()) + .anyMatch(name -> name.equalsIgnoreCase(contentName) || contentName.isBlank()); } /** * Wait for Solr to finish indexing: Indexing has caught up = true if search returns appropriate results - * - * @param userQuery: search query, this can include the fieldname, unique search string will guarantee accurate results + * + * @param userQuery: search query, this can include the fieldname, unique search string will guarantee accurate results * @param expectedInResults, true if entry is expected in the results set * @return true (indexing is finished) if search returns appropriate results */ @@ -307,7 +312,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * waitForIndexing method that matches / waits for filename, metadata to be indexed. - * + * * @param userQuery * @param expectedInResults * @return @@ -320,7 +325,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * waitForIndexing method that matches / waits for content to be indexed, this can take longer than metadata indexing. * Since Metadata is indexed first, use this method where tests, queries need content to be indexed too. - * + * * @param userQuery * @param expectedInResults * @return @@ -332,9 +337,9 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Wait for Solr to finish indexing: Indexing has caught up = true if search returns appropriate results - * - * @param fieldName: specific field to search for, e.g. name. When specified, the query will become: name:'userQuery' - * @param userQuery: search string, unique search string will guarantee accurate results + * + * @param fieldName: specific field to search for, e.g. name. When specified, the query will become: name:'userQuery' + * @param userQuery: search string, unique search string will guarantee accurate results * @param expectedInResults, true if entry is expected in the results set * @return true (indexing is finished) if search returns appropriate results */ @@ -347,7 +352,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Run a search as admin user and return the response - * + * * @param queryString: string to search for, unique search string will guarantee accurate results * @return the search response from the API */ @@ -358,8 +363,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Run a search as given user and return the response - * - * @param user: UserModel for the user you wish to run the query as + * + * @param user: UserModel for the user you wish to run the query as * @param queryString: string to search for, unique search string will guarantee accurate results * @return the search response from the API */ @@ -371,11 +376,11 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont searchRequest.setQuery(queryModel); return restClient.authenticateUser(user).withSearchAPI().search(searchRequest); } - + /** * Run a search with Spellcheck as given user and return the response - * @param user UserModel for the user you wish to run the query as - * @param queryModel The queryModel to search for, containing the query + * @param user UserModel for the user you wish to run the query as + * @param queryModel The queryModel to search for, containing the query * @param spellcheckQuery The Spellcheck Model containing the query * @return the search response from the API */ @@ -416,31 +421,22 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont query.setQuery(queryReq); return query; } - + protected DataUser getDataUser() { return dataUser; } - + /** * Helper method to test if the search query works and count matches where provided - * @param query: AFTS or cmis query string + * + * @param query: AFTS or cmis query string * @param expectedCount: Only successful response is checked, when expectedCount is null (can not be exactly specified), * @return SearchResponse */ protected SearchResponse testSearchQuery(String query, Integer expectedCount, SearchLanguage queryLanguage) { - RestRequestQueryModel queryModel = new RestRequestQueryModel(); - queryModel.setQuery(query); - - if (ofNullable(queryLanguage).isPresent()) - { - queryModel.setLanguage(queryLanguage.toString()); - } - - SearchResponse response = queryAsUser(testUser, queryModel); - - restClient.assertStatusCodeIs(HttpStatus.OK); + SearchResponse response = performSearch(testUser, query, queryLanguage); if (ofNullable(expectedCount).isPresent()) { @@ -449,7 +445,64 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont return response; } + + /** + * Helper method to test if the search query returns the expected results in the order given. + * + * @param query: AFTS or cmis query string + * @param expectedNames: The ordered list of names expected to be returned, + * @return SearchResponse + */ + protected SearchResponse testSearchQueryOrdered(String query, List expectedNames, SearchLanguage queryLanguage) + { + SearchResponse response = performSearch(testUser, query, queryLanguage); + + List names = response.getEntries().stream().map(s -> s.getModel().getName()).collect(Collectors.toList()); + + // Include lists in failure message as TestNG won't do this for lists. + assertEquals(names, expectedNames, "Unexpected results for query: " + query + " Expected: " + expectedNames + " but got " + names); + return response; + } + + /** + * Helper method to test if the search query returns the expected set of results. + * + * @param query: AFTS or cmis query string + * @param expectedNames: The ordered list of names expected to be returned, + * @return SearchResponse + */ + protected SearchResponse testSearchQueryUnordered(String query, Set expectedNames, SearchLanguage queryLanguage) + { + SearchResponse response = performSearch(testUser, query, queryLanguage); + + Set names = response.getEntries().stream().map(s -> s.getModel().getName()).collect(Collectors.toSet()); + + assertEquals(names, expectedNames, "Unexpected results for query: " + query); + + return response; + } + + private SearchResponse performSearch(UserModel asUser, String query, SearchLanguage queryLanguage) + { + RestRequestQueryModel queryModel = new RestRequestQueryModel(); + queryModel.setQuery(query); + + if (!ofNullable(asUser).isPresent()) + { + asUser = testUser; + } + + if (ofNullable(queryLanguage).isPresent()) + { + queryModel.setLanguage(queryLanguage.toString()); + } + + SearchResponse response = queryAsUser(asUser, queryModel); + + return response; + } + /** * Method to create and run a simple spellcheck query * When a spellcheck query is run a user, the query inputed and the user query is inputted @@ -461,8 +514,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont { RestRequestSpellcheckModel spellCheck = new RestRequestSpellcheckModel(); spellCheck.setQuery(userQuery); - - UserModel searchUser = ofNullable(user).isPresent()? user: testUser; + + UserModel searchUser = ofNullable(user).isPresent() ? user : testUser; SearchRequest searchReq = new SearchRequest(); RestRequestQueryModel queryReq = new RestRequestQueryModel(); queryReq.setQuery(query); @@ -475,8 +528,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Method to check the spellcheck object returned in the Search Response - * @param response SearchResponse - * @param spellCheckType String Values: searchInsteadFor, didYouMean or null + * @param response SearchResponse + * @param spellCheckType String Values: searchInsteadFor, didYouMean or null * @param spellCheckSuggestion String Values: suggestion string or null */ public void testSearchSpellcheckResponse(SearchResponse response, String spellCheckType, String spellCheckSuggestion) @@ -495,6 +548,6 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont { response.getContext().assertThat().field("spellCheck").isNotEmpty(); response.getContext().getSpellCheck().assertThat().field("suggestions").contains(spellCheckSuggestion); - } + } } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java new file mode 100644 index 000000000..987f141d1 --- /dev/null +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java @@ -0,0 +1,326 @@ +/* + * Copyright 2019 Alfresco Software, Ltd. All rights reserved. + * License rights for this program may be obtained from Alfresco Software, Ltd. + * pursuant to a written agreement and any use of this program without such an + * agreement is prohibited. + */ + +package org.alfresco.test.search.functional.searchServices.search; + +import static java.util.List.of; + +import static jersey.repackaged.com.google.common.collect.Sets.newHashSet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.alfresco.rest.model.RestNodeAssociationModelCollection; +import org.alfresco.rest.model.RestNodeChildAssociationModel; +import org.alfresco.rest.search.SearchResponse; +import org.alfresco.search.TestGroup; +import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; +import org.alfresco.utility.model.FileModel; +import org.alfresco.utility.model.FileType; +import org.alfresco.utility.model.FolderModel; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Test class tests AFTS Search In Field works + * Created for Search-840 + * + * @author Meenal Bhave + */ +public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest +{ + private FolderModel folder1, folder2; + private FileModel file1, file2, file3, file4, file5; + + @BeforeClass(alwaysRun = true) + public void dataPreparation() + { + // Create Folders: + // Folder1: Expected to be found with file.txt + folder1 = new FolderModel("file txt folder"); + dataContent.usingUser(testUser).usingSite(testSite).createFolder(folder1); + + // Folder2: Not expected to be found with file.txt + folder2 = new FolderModel("txt files folder"); + dataContent.usingUser(testUser).usingSite(testSite).createFolder(folder2); + + // Create File(s): Expected to be found with file.txt + file1 = new FileModel("file.txt", "file.txt", "", FileType.TEXT_PLAIN, "file.txt"); + + file2 = new FileModel("1-file.txt", "1-file.txt", "", FileType.TEXT_PLAIN, "1-file.txt"); + + file3 = new FileModel("file1.txt", "file1.txt", "", FileType.TEXT_PLAIN, "file1.txt"); + + file4 = new FileModel("txt file", "txt file", "", FileType.TEXT_PLAIN, "txt file"); + + // Not Expected to be found with file.txt + file5 = new FileModel("txt files", "txt files", "", FileType.TEXT_PLAIN, "txt files"); + + of(file1, file2, file3, file4, file5).forEach( + f -> dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder1).createContent(f)); + + waitForContentIndexing(file5.getContent(), true); + } + + @Test(priority = 1, groups = { TestGroup.ACS_63n }) + public void testSearchInFieldName() + { + // Field names in various formats + Stream fieldNames = Stream.of("{http://www.alfresco.org/model/content/1.0}name", + "@{http://www.alfresco.org/model/content/1.0}name", + "cm_name", + "cm:name", + "@cm:name", + "name"); + + // For each field name, check that queries return consistent results with / out '' + fieldNames.forEach(fieldName -> + { + // Query string without quotes + String query = fieldName + ":file.txt"; + + Set expectedNames = newHashSet(); + expectedNames.add("file.txt"); // file1 + expectedNames.add("1-file.txt"); // file2 + expectedNames.add("file1.txt"); // file3 + expectedNames.add("txt file"); // file4 + expectedNames.add("file txt folder"); // folder1 + + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + + // Query string in single quotes + query = fieldName + ":'file.txt'"; + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + }); + } + + @Test(priority = 2, groups = { TestGroup.ACS_63n }) + public void testSearchInFieldTitle() + { + // Field names in various formats + Stream fieldNames = Stream.of("{http://www.alfresco.org/model/content/1.0}title", + "@{http://www.alfresco.org/model/content/1.0}title", + "cm_title", + "cm:title", + "@cm:title"); + + // For each field name, check that queries return consistent results with / out '' + fieldNames.forEach(fieldName -> + { + String query = fieldName + ":" + file2.getName(); + boolean fileFound = isContentInSearchResults(query, file2.getName(), true); + Assert.assertTrue(fileFound, "File Not found for query: " + query); + + testSearchQuery(query, 1, SearchLanguage.AFTS); + + query = fieldName + ":'" + file2.getName() + "\'"; + fileFound = isContentInSearchResults(query, file2.getName(), true); + Assert.assertTrue(fileFound, "File Not found for query: " + query); + + testSearchQuery(query, 1, SearchLanguage.AFTS); + }); + } + + @Test(priority = 3, groups = { TestGroup.ACS_63n }) + public void testSearchInFieldContent() + { + // Field names in various formats + List fieldNames = new ArrayList<>(); + fieldNames.add("TEXT"); + fieldNames.add("{http://www.alfresco.org/model/dictionary/1.0}content"); + fieldNames.add("cm:content"); + fieldNames.add("d:content"); + + // For each field name, check that queries return consistent results with / out '' + fieldNames.forEach(fieldName -> + { + String query = fieldName + ":" + file3.getContent(); + boolean fileFound = isContentInSearchResults(query, file3.getName(), true); + Assert.assertTrue(fileFound, "File Not found for query: " + query); + + Integer resultCount1 = testSearchQuery(query, null, SearchLanguage.AFTS).getPagination().getTotalItems(); + + query = fieldName + ":'" + file3.getContent() + "\'"; + fileFound = isContentInSearchResults(query, file3.getName(), true); + Assert.assertTrue(fileFound, "File Not found for query: " + query); + + testSearchQuery(query, resultCount1, SearchLanguage.AFTS).getPagination().getTotalItems(); + }); + } + + @Test(priority = 4) + public void testSearchInFieldSITE() + { + String query = "SITE:" + testSite.getId(); + boolean fileFound = isContentInSearchResults(query, folder1.getName(), true); + Assert.assertTrue(fileFound, "Site Not found for query: " + query); + + Integer resultCount1 = testSearchQuery(query, 8, SearchLanguage.AFTS).getPagination().getTotalItems(); + + query = "SITE:'" + testSite.getId() + "\'"; + fileFound = isContentInSearchResults(query, folder1.getName(), true); + Assert.assertTrue(fileFound, "Site Not found for query: " + query); + + testSearchQuery(query, resultCount1, SearchLanguage.AFTS).getPagination().getTotalItems(); + } + + @Test(priority = 5) + public void testSearchInFieldTYPE() + { + // Field names in various formats + List fieldNames = new ArrayList<>(); + fieldNames.add("TYPE"); + fieldNames.add("EXACTTYPE"); + + // For each field name, check that queries return consistent results with / out '' + fieldNames.forEach(fieldName -> { + + String query = fieldName + ":cm\\:content" + " and =cm:name:" + file1.getName(); + boolean fileFound = isContentInSearchResults(query, file1.getName(), true); + Assert.assertTrue(fileFound, "Content Not found for query: " + query); + + Integer resultCount1 = testSearchQuery(query, 1, SearchLanguage.AFTS).getPagination().getTotalItems(); + + query = fieldName + ":'cm:content'" + " and =cm:name:" + file1.getName(); + fileFound = isContentInSearchResults(query, file1.getName(), true); + Assert.assertTrue(fileFound, "Content Not found for query: " + query); + + testSearchQuery(query, resultCount1, SearchLanguage.AFTS).getPagination().getTotalItems(); + }); + } + + @Test(priority = 6) + public void testSearchInFieldID() throws Exception + { + String query = "ID:'workspace://SpacesStore/" + file1.getNodeRefWithoutVersion() + "'"; + boolean fileFound = isContentInSearchResults(query, file1.getName(), true); + Assert.assertTrue(fileFound, "Content Not found for query: " + query); + + testSearchQuery(query, 1, SearchLanguage.AFTS).getPagination().getTotalItems(); + } + + @Test(priority = 7) + public void testSearchInFieldPARENT() + { + String query = "PARENT:" + folder1.getNodeRefWithoutVersion(); + + Set expectedNames = newHashSet(); + expectedNames.add(file1.getName()); + expectedNames.add(file2.getName()); + expectedNames.add(file3.getName()); + expectedNames.add(file4.getName()); + expectedNames.add(file5.getName()); + + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + + query = "PARENT:'" + folder1.getNodeRefWithoutVersion() + "\'"; + + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + } + + @Test(priority = 8) + public void testSearchInFieldPRIMARYPARENT() throws Exception + { + // Create Secondary association in testFolder2 + RestNodeChildAssociationModel childAssoc1 = new RestNodeChildAssociationModel(file1.getNodeRefWithoutVersion(), "cm:contains"); + String secondaryChildrenBody = "[" + childAssoc1.toJson() + "]"; + + restClient.authenticateUser(testUser).withCoreAPI().usingResource(folder2).createSecondaryChildren(secondaryChildrenBody); + RestNodeAssociationModelCollection secondaryChildren = restClient.authenticateUser(testUser).withCoreAPI().usingResource(folder2).getSecondaryChildren(); + secondaryChildren.getEntryByIndex(0).assertThat().field("id").is(file1.getNodeRefWithoutVersion()); + + String query = "PRIMARYPARENT:'workspace://SpacesStore/" + folder1.getNodeRef() + "'"; + + Set expectedNames = newHashSet(); + expectedNames.add(file1.getName()); + expectedNames.add(file2.getName()); + expectedNames.add(file3.getName()); + expectedNames.add(file4.getName()); + expectedNames.add(file5.getName()); + + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + + query = "PARENT:'workspace://SpacesStore/" + folder2.getNodeRef() + "'"; + boolean fileFound = isContentInSearchResults(query, file1.getName(), true); + Assert.assertTrue(fileFound, "Expected Content Not found for query: " + query); + + testSearchQuery(query, 1, SearchLanguage.AFTS); + } + + @Test(priority = 9, groups = { TestGroup.ACS_63n }) + public void testSearchInFieldNameExactMatch() + { + // Check that queries return consistent results with / out '' + String query = "=name:" + file1.getName(); + boolean fileFound = isContentInSearchResults(query, file1.getName(), true); + Assert.assertTrue(fileFound, "File Not found for query: " + query); + + Integer resultCount1 = testSearchQuery(query, 1, SearchLanguage.AFTS).getPagination().getTotalItems(); + Assert.assertSame(resultCount1, 1, "File count does not match for query: " + query); + + query = "=name:'" + file1.getName() + "\'"; + fileFound = isContentInSearchResults(query, file1.getName(), true); + Assert.assertTrue(fileFound, "File Not found for query: " + query); + + testSearchQuery(query, resultCount1, SearchLanguage.AFTS).getPagination().getTotalItems(); + } + + @Test(priority = 10, groups = { TestGroup.ACS_63n }) + public void testSearchInFieldNameQueryExpansion() + { + // Check that queries return consistent results with / out '' + String query = "~name:" + file1.getName(); + + Set expectedNames = newHashSet(); + expectedNames.add(file1.getName()); + expectedNames.add(file2.getName()); + expectedNames.add(file3.getName()); + expectedNames.add(file4.getName()); + expectedNames.add(folder1.getName()); + + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + + query = "~name:'" + file1.getName() + "\'"; + testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + } + + @Test(priority = 11, groups = { TestGroup.ACS_63n }) + public void testWithConjunctionDisjunctionAndNegation() + { + // Query string to include Conjunction, Disjunction and Negation + + String query1 = "~name:" + file1.getName(); // Query expected to return 5 results + String query2 = "=name:" + file2.getName(); // Query expected to return 1 result + String query3 = "=name:" + file3.getName(); // Query expected to return 1 result + + // Check Query with Conjunction, Negation, Disjunction: returns right results + // "~name:file.txt and ! (=name:1-file.txt or =name:file1.txt)" + String query = query1 + " and ! (" + query2 + " or " + query3 + ")"; + + // Check that expected files are included in the results + Set expectedNames = newHashSet(); + expectedNames.add("file.txt"); // file1 + expectedNames.add("txt file"); // file4 + expectedNames.add("file txt folder"); // folder1 + + SearchResponse response = testSearchQueryUnordered(query, expectedNames, SearchLanguage.AFTS); + + // Check result count is 5-(1+1)=3 + int resultCount = response.getPagination().getTotalItems(); + Assert.assertEquals(resultCount, 3, "File count does not match for query: " + query); + + // Check that file2 and file3 are excluded from the results + boolean fileFound = isContentInSearchResponse(response, file2.getName()); + Assert.assertFalse(fileFound, "File2 found for query: " + query); + + fileFound = isContentInSearchResponse(response, file3.getName()); + Assert.assertFalse(fileFound, "File3 found for query: " + query); + } +} From 80e31a85753ba6d7f6513869f899d37be1f43ee6 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 9 Dec 2019 14:17:48 +0100 Subject: [PATCH 040/129] [SEARCH-1994] some fixes to make contentstore replication working on windows --- .../alfresco/solr/handler/AlfrescoIndexFetcher.java | 11 +++++------ .../solr/handler/ContentStoreReplicationIT.java | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 477a38dec..81e40fef3 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1703,9 +1703,7 @@ class AlfrescoIndexFetcher try { Files.createDirectories(Paths.get(csFile.getParent())); - if (!tmpFile.renameTo(csFile)){ - throw new RuntimeException("Failed while moving content store file to " + csFile.getAbsolutePath()); - } + Files.copy(tmpFile.toPath(), csFile.toPath()); } catch (IOException e) { @@ -1735,7 +1733,7 @@ class AlfrescoIndexFetcher * Deletes from contentstore all the files that has not been updated. * @param contentStorePath */ - private void cleanUpContentStore(String contentStorePath) + private void cleanUpContentStore(String contentStorePath) throws Exception { AtomicInteger fileDeleted = new AtomicInteger(); Set fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME)) @@ -1744,7 +1742,7 @@ class AlfrescoIndexFetcher { Files.walk(Paths.get(contentStorePath)).forEach(p -> { File f = new File(p.toUri()); - if (!f.isDirectory() && !fileNames.contains(p.toString().replaceFirst(contentStorePath, ""))) + if (!f.isDirectory() && !fileNames.contains(p.toString().replace(contentStorePath, ""))) { try { @@ -1758,9 +1756,10 @@ class AlfrescoIndexFetcher } }); } - catch (IOException e) + catch (Exception e) { LOG.error("Impossible to delete unnecessary files. Content store may contains unused contents"); + throw(e); } LOG.info("deleted {} unnecessary files from content store", fileDeleted); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java index 8fb8eeb58..678afd5ad 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/handler/ContentStoreReplicationIT.java @@ -141,8 +141,8 @@ public class ContentStoreReplicationIT extends AbstractAlfrescoDistributedIT { master.stop(); slave.stop(); - FileUtils.forceDelete(new File(masterSolrHome.getParent().toUri())); - FileUtils.forceDelete(new File(slaveSolrHome.getParent().toUri())); + FileUtils.deleteQuietly(new File(masterSolrHome.getParent().toUri())); + FileUtils.deleteQuietly(new File(slaveSolrHome.getParent().toUri())); SOLRAPIQueueClient.nodeMetaDataMap.clear(); SOLRAPIQueueClient.transactionQueue.clear(); From f908b11d49ff7ee9f3afd1ca2ae1ed208c400f2e Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 9 Dec 2019 14:31:46 +0100 Subject: [PATCH 041/129] [SEARCH-1994] replace existend file when copy tmp contentstore files --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 81e40fef3..66d676867 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1703,7 +1703,7 @@ class AlfrescoIndexFetcher try { Files.createDirectories(Paths.get(csFile.getParent())); - Files.copy(tmpFile.toPath(), csFile.toPath()); + Files.copy(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { From 0b63a150f3726c70fb84e9ab10c36993eca8a410 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2019 22:14:29 +0000 Subject: [PATCH 042/129] Bump randomizedtesting-runner from 2.7.4 to 2.7.5 in /search-services Bumps randomizedtesting-runner from 2.7.4 to 2.7.5. Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 3a1a3401d..a348e4fe5 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -171,7 +171,7 @@ com.carrotsearch.randomizedtesting randomizedtesting-runner - 2.7.4 + 2.7.5 test From 0d1af08d8f427c80c5065a783aba3e924178cd92 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 10 Dec 2019 10:22:57 +0000 Subject: [PATCH 043/129] SEARCH-2001 Remove transitive dependency of old TAS restapi test artifact. This was introduced by accident when the artifact name changed and so we stopped overriding the artifact from GS. --- e2e-test/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index c7911ea2a..e14adef45 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -71,6 +71,12 @@ alfresco-governance-services-automation-enterprise-rest-api ${rm.version} test + + + org.alfresco.tas + restapi-test + + com.fasterxml.jackson.core @@ -88,6 +94,10 @@ com.fasterxml.jackson.core jackson-databind + + org.alfresco.tas + restapi-test + From 6f2a0883f57420ba89b538079bcb5cf7efc9b10f Mon Sep 17 00:00:00 2001 From: agazzarini Date: Wed, 11 Dec 2019 09:39:05 +0100 Subject: [PATCH 044/129] [ SEARCH-1506 ] Stream tests load data only once --- .../solr/AbstractAlfrescoDistributedIT.java | 133 ++++++---------- .../org/alfresco/solr/SolrITInitializer.java | 145 ++++++++---------- 2 files changed, 113 insertions(+), 165 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 3f6cdffe9..77c6d5f58 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -33,7 +33,6 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.junit.Before; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,8 +82,9 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSI * class hierarchy. Ideally this function should be retired in favour of better * annotations.. * - * @since solr 1.5 - * @author Michael Suzuki + * @since solr 1.4.1 + * @author Michael Suzuki + * @author Andrea Gazzarini */ @ThreadLeakLingering(linger = 5000) public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer @@ -92,8 +92,8 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer protected static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected String[] deadServers; - protected static SolrResponsesComparator solrComparator = new SolrResponsesComparator(); - protected static RandomSupplier solrRandomSupplier; + protected static SolrResponsesComparator SOLR_RESPONSE_COMPARATOR = new SolrResponsesComparator(); + protected static RandomSupplier SOLR_RANDOM_SUPPLIER; // to stress with higher thread counts and requests, make sure the junit // xml formatter is not being used (all output will be buffered before @@ -112,34 +112,26 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer * Test configs may use the ${hostContext} variable to access * this system property. *

- * */ @BeforeClass public static void setUpSolrTestProperties() { + SOLR_RANDOM_SUPPLIER = new RandomSupplier(); System.setProperty("alfresco.test", "true"); System.setProperty("solr.tests.maxIndexingThreads", "10"); System.setProperty("solr.tests.ramBufferSizeMB", "1024"); } - @Before - public void setupPerTest() - { - this.solrRandomSupplier = new RandomSupplier(); - } - - - public static String[] fieldNames = new String[] - { "n_ti1", "n_f1", "n_tf1", "n_d1", "n_td1", "n_l1", "n_tl1", "n_dt1", "n_tdt1" }; - + public static final String[] FIELD_NAMES = new String[] { "n_ti1", "n_f1", "n_tf1", "n_d1", "n_td1", "n_l1", "n_tl1", "n_dt1", "n_tdt1" }; protected static String[] getFieldNames() { - return fieldNames; + return FIELD_NAMES; } - protected void putHandleDefaults() { - solrComparator.putHandleDefaults(); + protected static void putHandleDefaults() + { + SOLR_RESPONSE_COMPARATOR.putHandleDefaults(); } /** @@ -147,8 +139,6 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer * @param query - query to execute * @param count - min number of results each shard must satisfy * @param waitMillis - total ms to wait - * @return - * @throws Exception */ public static boolean checkMinCountPerShard(Query query, int count, long waitMillis) throws SolrServerException,IOException { @@ -160,7 +150,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer for (SolrClient singleShard : shardedClients) { allShardCompliant = false; - int totalHits = 0; + int totalHits; int cycles = 1; while ((new Date()).getTime() < timeout && (!allShardCompliant)) { @@ -172,11 +162,11 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer } try { - Thread.sleep((long) (500 * cycles++)); + Thread.sleep(500 * cycles++); } catch (InterruptedException e) { - continue; + // Ignore } } } @@ -205,11 +195,6 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer /** * Waits until all cores (including shards) reach a count. - * - * @param query - * @param count - * @param waitMillis - * @throws Exception */ public static void waitForDocCountAllCores(Query query, int count, long waitMillis) throws Exception { @@ -230,12 +215,9 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer /** * Delele by query on all Clients - * - * @param q - * @throws Exception */ - public static void deleteByQueryAllClients(String q) throws Exception { - + public static void deleteByQueryAllClients(String q) throws Exception + { List clients = getStandaloneAndShardedClients(); for (SolrClient client : clients) { @@ -245,14 +227,11 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer /** * Gets the Default test client. - * - * @return */ protected static SolrClient getDefaultTestClient() { return solrCollectionNameToStandaloneClient.get(DEFAULT_TEST_CORENAME); } - protected static List getShardedClients() { @@ -266,7 +245,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer */ public static List getStandaloneAndShardedClients() { - List clients = new ArrayList(); + List clients = new ArrayList<>(); clients.addAll(solrCollectionNameToStandaloneClient.values()); clients.addAll(clientShards); return clients; @@ -275,17 +254,11 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer public static List getStandaloneClients() { - List clients = new ArrayList(); - clients.addAll(solrCollectionNameToStandaloneClient.values()); - return clients; + return new ArrayList<>(solrCollectionNameToStandaloneClient.values()); } /** * Waits for the doc count on the first core available, then checks all the Shards match. - * @param query - * @param count - * @param waitMillis - * @throws Exception */ public static void waitForDocCount(Query query, int count, long waitMillis) throws Exception { @@ -332,7 +305,8 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer return escapedField + ":" + value + " "; } - protected static String escapeQueryChars(String query) { + protected static String escapeQueryChars(String query) + { return query.replaceAll("\\:", "\\\\:") .replaceAll("\\{", "\\\\{") .replaceAll("\\}", "\\\\}"); @@ -341,7 +315,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer public static SolrQuery luceneToSolrQuery(Query query) { String[] terms = query.toString().split(" "); - String escapedQuery = new String(); + String escapedQuery = ""; for (String t : terms) { escapedQuery += escapeQueryClause(t); @@ -366,8 +340,10 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer waitForShardsCount(solrQuery, count, waitMillis, start); } - protected static void injectDocToShards(long txnId, long aclId, long dbId, String owner) throws Exception { - for(SolrClient clientShard : clientShards) { + protected static void injectDocToShards(long txnId, long aclId, long dbId, String owner) throws Exception + { + for(SolrClient clientShard : clientShards) + { SolrInputDocument doc = new SolrInputDocument(); String id = AlfrescoSolrDataModel.getNodeDocumentId(AlfrescoSolrDataModel.DEFAULT_TENANT, aclId, dbId); doc.addField(FIELD_SOLR4_ID, id); @@ -383,21 +359,20 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer /** * Gets the cores for the jetty instances - * @return */ protected static List getJettyCores(Collection runners) { - List cores = new ArrayList(); + List cores = new ArrayList<>(); for (JettySolrRunner jettySolrRunner : runners) { - jettySolrRunner.getCoreContainer().getCores().forEach(aCore -> cores.add(aCore)); + cores.addAll(jettySolrRunner.getCoreContainer().getCores()); } return cores; } protected static List getAdminHandlers(Collection runners) { - List coreAdminHandlers = new ArrayList(); + List coreAdminHandlers = new ArrayList<>(); for (JettySolrRunner jettySolrRunner : runners) { CoreContainer coreContainer = jettySolrRunner.getCoreContainer(); @@ -418,32 +393,27 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer List clients = getShardedClients(); SolrQuery query = luceneToSolrQuery(new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_NODE))); StringBuilder error = new StringBuilder(); - for (int i = 0; i < clients.size(); ++i) + for (SolrClient client : clients) { - SolrClient client = clients.get(i); - - QueryResponse response = client.query(query); int totalHits = (int) response.getResults().getNumFound(); - if(totalHits > 0) + if (totalHits > 0) { shardHit++; } - if(totalHits < count) + + if (totalHits < count) { - if (ignoreZero && totalHits == 0) - { - log.info(client+": have zero hits "); - } - else - { - error.append(" "+client+": "); - error.append("Expected nodes per shard greater than "+count+" found "+totalHits+" : "+query.toString()); + if (ignoreZero && totalHits == 0) { + log.info(client + ": have zero hits "); + } else { + error.append(" " + client + ": "); + error.append("Expected nodes per shard greater than " + count + " found " + totalHits + " : " + query.toString()); } } - log.info(client+": Hits "+totalHits); + log.info(client + ": Hits " + totalHits); } @@ -560,7 +530,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer protected String getShardsString() { - Random r = solrRandomSupplier.getRandomGenerator(); + Random r = SOLR_RANDOM_SUPPLIER.getRandomGenerator(); if (deadServers == null) return shards; @@ -610,7 +580,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer protected static SolrInputDocument addRandFields(SolrInputDocument sdoc) { - addFields(sdoc, solrRandomSupplier.getRandFields(getFieldNames(), solrRandomSupplier.getRandValues())); + addFields(sdoc, SOLR_RANDOM_SUPPLIER.getRandFields(getFieldNames(), SOLR_RANDOM_SUPPLIER.getRandValues())); return sdoc; } @@ -662,7 +632,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer { UpdateResponse controlRsp = add(client1, params, sdocs); UpdateResponse specificRsp = add(client2, params, sdocs); - solrComparator.compareSolrResponses(specificRsp, controlRsp); + SOLR_RESPONSE_COMPARATOR.compareSolrResponses(specificRsp, controlRsp); return specificRsp; } @@ -724,9 +694,6 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer /** * * Commits to the specified client, and optionally all shards - * @param client - * @param andShards - * @throws Exception */ protected static void commit(SolrClient client, boolean andShards) throws Exception { @@ -743,7 +710,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer protected static QueryResponse queryRandomShard(ModifiableSolrParams params) throws SolrServerException, IOException { - Random r = solrRandomSupplier.getRandomGenerator(); + Random r = SOLR_RANDOM_SUPPLIER.getRandomGenerator(); int which = r.nextInt(clientShards.size()); SolrClient client = clientShards.get(which); QueryResponse rsp = client.query(params); @@ -755,13 +722,13 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer params.set("distrib", "false"); QueryRequest request = getAlfrescoRequest(json, params); QueryResponse controlRsp = request.process(solrClient); - solrComparator.validateResponse(controlRsp); + SOLR_RESPONSE_COMPARATOR.validateResponse(controlRsp); if (andShards) { params.remove("distrib"); setDistributedParams(params); QueryResponse rsp = queryRandomShard(json, params); - solrComparator.compareResponses(rsp, controlRsp); + SOLR_RESPONSE_COMPARATOR.compareResponses(rsp, controlRsp); return rsp; } else @@ -772,7 +739,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer protected static QueryResponse queryRandomShard(String json, SolrParams params) throws SolrServerException, IOException { - Random r = solrRandomSupplier.getRandomGenerator(); + Random r = SOLR_RANDOM_SUPPLIER.getRandomGenerator(); int which = r.nextInt(clientShards.size()); SolrClient client = clientShards.get(which); QueryRequest request = getAlfrescoRequest(json, params); @@ -791,13 +758,13 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer */ protected QueryResponse query(SolrClient solrClient, boolean setDistribParams, SolrParams p) throws Exception { - Random r = solrRandomSupplier.getRandomGenerator(); + Random r = SOLR_RANDOM_SUPPLIER.getRandomGenerator(); final ModifiableSolrParams params = new ModifiableSolrParams(p); // TODO: look into why passing true causes fails params.set("distrib", "false"); final QueryResponse controlRsp = solrClient.query(params); - solrComparator.validateResponse(controlRsp); + SOLR_RESPONSE_COMPARATOR.validateResponse(controlRsp); params.remove("distrib"); if (setDistribParams) @@ -805,7 +772,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer QueryResponse rsp = queryRandomShard(params); - solrComparator.compareResponses(rsp, controlRsp); + SOLR_RESPONSE_COMPARATOR.compareResponses(rsp, controlRsp); if (stress > 0) { @@ -827,7 +794,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer QueryResponse rsp = client.query(new ModifiableSolrParams(params)); if (verifyStress) { - solrComparator.compareResponses(rsp, controlRsp); + SOLR_RESPONSE_COMPARATOR.compareResponses(rsp, controlRsp); } } catch (SolrServerException | IOException e) { @@ -865,7 +832,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer first = rsp; } else { - solrComparator.compareResponses(first, rsp); + SOLR_RESPONSE_COMPARATOR.compareResponses(first, rsp); } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index b06df1a09..3bc5bd666 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -91,8 +91,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 //Standalone Tests protected static SolrCore defaultCore; - protected static final int clientConnectionTimeout = DEFAULT_CONNECTION_TIMEOUT;; - protected static final int clientSoTimeout = 90000;; + protected static final int clientConnectionTimeout = DEFAULT_CONNECTION_TIMEOUT; + protected static final int clientSoTimeout = 90000; protected static final String id = "id"; @@ -129,21 +129,22 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 jettyContainers = new HashMap<>(); nodeCnt = new AtomicInteger(0); - String serverName = testClassName; - currentTestName = serverName; + currentTestName = testClassName; String[] coreNames = new String[]{DEFAULT_TEST_CORENAME}; - distribSetUp(serverName); - RandomSupplier.RandVal.uniqueValues = new HashSet(); // reset random values - createServers(serverName, coreNames, numShards,solrcoreProperties); - System.setProperty("solr.solr.home", testDir.toPath().resolve(serverName).toString()); + distribSetUp(testClassName); + RandomSupplier.RandVal.uniqueValues = new HashSet<>(); // reset random values + createServers(testClassName, coreNames, numShards,solrcoreProperties); + System.setProperty("solr.solr.home", testDir.toPath().resolve(testClassName).toString()); } private static Properties addExplicitShardingProperty(Properties solrcoreProperties) { - if(solrcoreProperties == null){ + if(solrcoreProperties == null) + { solrcoreProperties = new Properties(); } + if(solrcoreProperties.getProperty("shard.method")==null) { solrcoreProperties.put("shard.method", "EXPLICIT_ID"); @@ -151,7 +152,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 return solrcoreProperties; } - public static void initSingleSolrServer(String testClassName, Properties solrcoreProperties) throws Throwable { + public static void initSingleSolrServer(String testClassName, Properties solrcoreProperties) throws Throwable + { initSolrServers(0,testClassName,solrcoreProperties); JettySolrRunner jsr = jettyContainers.get(testClassName); @@ -163,7 +165,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { int i = 0; extras = new String[solrcoreProperties.size()*2]; - for (Map.Entry prop:solrcoreProperties.entrySet()) { + for (Map.Entry prop:solrcoreProperties.entrySet()) + { extras[i++] = "property."+prop.getKey(); extras[i++] = (String) prop.getValue(); } @@ -185,7 +188,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 destroyServers(); distribTearDown(); - boolean keepTests = Boolean.valueOf(System.getProperty("keep.tests")); + boolean keepTests = Boolean.parseBoolean(System.getProperty("keep.tests")); if (!keepTests) FileUtils.deleteDirectory(testDir); } catch (Exception e) @@ -203,7 +206,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 return System.getProperty("user.dir") + "/target/test-classes/test-files"; } - public static void distribSetUp(String serverName) throws Exception + public static void distribSetUp(String serverName) { SolrTestCaseJ4.resetExceptionIgnores(); // ignore anything with // ignore_exception in it @@ -213,7 +216,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 System.setProperty("solr.log.dir", testDir.toPath().resolve(serverName).toString()); } - public static void distribTearDown() throws Exception + public static void distribTearDown() { System.clearProperty("solr.directoryFactory"); System.clearProperty("solr.log.dir"); @@ -230,8 +233,6 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 /** * Creates a JettySolrRunner (if one didn't exist already). DOES NOT START IT. - * @return - * @throws Exception */ protected static JettySolrRunner createJetty(String jettyKey, boolean basicAuth) throws Exception { @@ -243,19 +244,13 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { Path jettySolrHome = testDir.toPath().resolve(jettyKey); seedSolrHome(jettySolrHome); - JettySolrRunner jetty = createJetty(jettySolrHome.toFile(), null, null, false, 0, getSchemaFile(), basicAuth); - return jetty; + return createJetty(jettySolrHome.toFile(), null, null, false, 0, getSchemaFile(), basicAuth); } } /** * Adds the core config information to the jetty file system. * Its best to call this before calling start() on Jetty - * @param jettyKey - * @param sourceConfigName - * @param coreName - * @param additionalProperties - * @throws Exception */ protected static void addCoreToJetty(String jettyKey, String sourceConfigName, String coreName, Properties additionalProperties) throws Exception { @@ -289,17 +284,14 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 out.close(); in.close(); } - } - } /** * Starts jetty if its not already running - * @param jsr - * @throws Exception */ - protected static void startJetty(JettySolrRunner jsr) throws Exception { + protected static void startJetty(JettySolrRunner jsr) throws Exception + { if (!jsr.isRunning()) { jsr.start(); @@ -310,37 +302,40 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { boolean basicAuth = additionalProperties != null ? Boolean.parseBoolean(additionalProperties.getProperty("BasicAuth", "false")) : false; - JettySolrRunner jsr = createJetty(jettyKey, basicAuth); jettyContainers.put(jettyKey, jsr); Properties properties = new Properties(); - if(additionalProperties != null && additionalProperties.size() > 0) { + if(additionalProperties != null && additionalProperties.size() > 0) + { properties.putAll(additionalProperties); properties.remove("shard.method"); } - for (int i = 0; i < coreNames.length; i++) + for (String coreName : coreNames) { - addCoreToJetty(jettyKey, coreNames[i], coreNames[i], properties); + addCoreToJetty(jettyKey, coreName, coreName, properties); } //Now start jetty startJetty(jsr); int jettyPort = jsr.getLocalPort(); - for (int i = 0; i < coreNames.length; i++) + for (String coreName : coreNames) { - String url = buildUrl(jettyPort) + "/" + coreNames[i]; + String url = buildUrl(jettyPort) + "/" + coreName; + log.info(url); - solrCollectionNameToStandaloneClient.put(coreNames[i], createNewSolrClient(url)); + + solrCollectionNameToStandaloneClient.put(coreName, createNewSolrClient(url)); } shardsArr = new String[numShards]; StringBuilder sb = new StringBuilder(); - if (additionalProperties == null) { + if (additionalProperties == null) + { additionalProperties = new Properties(); } String[] ranges = {"0-100", "100-200", "200-300", "300-400"}; @@ -382,6 +377,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 solrHomes.add(jetty.getSolrHome()); jetty.stop(); } + for (SolrClient jClients : solrCollectionNameToStandaloneClient.values()) { jClients.close(); @@ -409,25 +405,16 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 solrCollectionNameToStandaloneClient.clear(); } - public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled, int port, String schemaOverride, boolean basicAuth) throws Exception + public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled, int port, String schemaOverride, boolean basicAuth) { return createJetty(solrHome, dataDir, shardList, sslEnabled, port, schemaOverride, useExplicitNodeNames, basicAuth); } /** * Create a solr jetty server. - * - * @param solrHome - * @param dataDir - * @param shardList - * @param port - * @param schemaOverride - * @param explicitCoreNodeName - * @return - * @throws Exception */ public static JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, boolean sslEnabled, int port, - String schemaOverride, boolean explicitCoreNodeName, boolean basicAuth) throws Exception + String schemaOverride, boolean explicitCoreNodeName, boolean basicAuth) { Properties props = new Properties(); if (schemaOverride != null) @@ -443,20 +430,21 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { props.setProperty("coreNodeName", Integer.toString(nodeCnt.incrementAndGet())); } + SSLConfig sslConfig = new SSLConfig(sslEnabled, false, null, null, null, null); - JettyConfig config = null; + JettyConfig config; - if(basicAuth) { - System.out.println("###### adding basic auth ######"); + if(basicAuth) + { + log.info("###### adding basic auth ######"); config = JettyConfig.builder().setContext("/solr").setPort(port).withFilter(BasicAuthFilter.class, "/sql/*").stopAtShutdown(true).withSSLConfig(sslConfig).build(); } else { - System.out.println("###### no basic auth ######"); + log.info("###### no basic auth ######"); config = JettyConfig.builder().setContext("/solr").setPort(port).stopAtShutdown(true).withSSLConfig(sslConfig).build(); } - JettySolrRunner jetty = new JettySolrRunner(solrHome.getAbsolutePath(), props, config); - return jetty; + return new JettySolrRunner(solrHome.getAbsolutePath(), props, config); } /** @@ -510,11 +498,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 */ protected static void seedSolrHome(Path jettyHome) throws IOException { - String solrxml = getSolrXml(); - if (solrxml != null) - { - FileUtils.copyFile(new File(getTestFilesHome(), solrxml), jettyHome.resolve(getSolrXml()).toFile()); - } + FileUtils.copyFile(new File(getTestFilesHome(), getSolrXml()), jettyHome.resolve(getSolrXml()).toFile()); + //Add solr home conf folder with alfresco based configuration. FileUtils.copyDirectory(new File(getTestFilesHome() + "/conf"), jettyHome.resolve("conf").toFile()); // Add alfresco data model def @@ -547,36 +532,26 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 FileUtils.copyDirectory(coreSourceConfig.resolve("conf").toFile(), confDir.toFile()); } - protected void setupJettySolrHome(String coreName, Path jettyHome) throws IOException + public static class BasicAuthFilter implements Filter { - seedSolrHome(jettyHome); - Properties coreProperties = new Properties(); - coreProperties.setProperty("name", coreName); - coreProperties.setProperty("shard", "${shard:}"); - coreProperties.setProperty("collection", "${collection:" + coreName + "}"); - coreProperties.setProperty("config", "${solrconfig:solrconfig.xml}"); - coreProperties.setProperty("schema", "${schema:schema.xml}"); - coreProperties.setProperty("coreNodeName", "${coreNodeName:}"); - - writeCoreProperties(jettyHome.resolve("cores").resolve(coreName), coreProperties, coreName); - } - - public static class BasicAuthFilter implements Filter { - - public BasicAuthFilter() { + public BasicAuthFilter() + { } - public void init(FilterConfig config) { + public void init(FilterConfig config) + { } public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) - throws IOException, ServletException { + throws IOException, ServletException + { //Parse the basic auth filter String auth = ((HttpServletRequest)request).getHeader("Authorization"); - if(auth != null) { + if(auth != null) + { auth = auth.replace("Basic ", ""); byte[] bytes = Base64.getDecoder().decode(auth); String decodedBytes = new String(bytes); @@ -584,18 +559,24 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 String user = pair[0]; String password = pair[1]; //Just look for the hard coded user and password. - if (user.equals("test") && password.equals("pass")) { + if (user.equals("test") && password.equals("pass")) + { filterChain.doFilter(request, response); - } else { + } + else + { ((HttpServletResponse) response).sendError(HttpServletResponse.SC_FORBIDDEN); } - } else { + } + else + { ((HttpServletResponse) response).sendError(HttpServletResponse.SC_FORBIDDEN); } } - public void destroy() { + public void destroy() + { } } -} +} \ No newline at end of file From d650cf497d0e65098827a9c15e3b8bbeec112a12 Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Wed, 11 Dec 2019 17:40:59 +0000 Subject: [PATCH 045/129] SEARCH-2011 Updating Slave replica config setup --- .../src/docker/6.x/docker-compose.yml | 25 +++++++++++ .../src/docker/search_config_setup.sh | 41 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/search-services/packaging/src/docker/6.x/docker-compose.yml b/search-services/packaging/src/docker/6.x/docker-compose.yml index 1174827d3..ca4268e79 100644 --- a/search-services/packaging/src/docker/6.x/docker-compose.yml +++ b/search-services/packaging/src/docker/6.x/docker-compose.yml @@ -37,6 +37,10 @@ services: search: image: quay.io/alfresco/search-services:${SEARCH_TAG} environment: + #Replication properties + - REPLICATION_TYPE=master + #- REPLICATION_AFTER=commit,startup- SOLR_ALFRESCO_HOST=alfresco + #- REPLICATION_CONFIG_FILES=schema.xml,stopwords.txt- SOLR_ALFRESCO_PORT=8080 #Solr needs to know how to register itself with Alfresco - SOLR_ALFRESCO_HOST=alfresco - SOLR_ALFRESCO_PORT=8080 @@ -51,6 +55,27 @@ services: - ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK} ports: - 8083:8983 #Browser port + #search_slave: + # image: quay.io/alfresco/search-services:${SEARCH_TAG} + # environment: + # #Replication properties + # - REPLICATION_TYPE=slave + # - REPLICATION_MASTER_HOST=search + # - REPLICATION_MASTER_PORT=8983 + # #- REPLICATION_MASTER_PROTOCOL=http + # #- REPLICATION_CORE_NAME=alfresco + # #- REPLICATION_POLL_INTERVAL=00:00:60 + # #Solr needs to know how to register itself with Alfresco + # - SOLR_ALFRESCO_HOST=alfresco + # - SOLR_ALFRESCO_PORT=8080 + # #Alfresco needs to know how to call solr + # - SOLR_SOLR_HOST=search + # - SOLR_SOLR_PORT=8983 + # #Create the default alfresco and archive cores + # - SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive + # ports: + # - 8084:8983 #Browser port + activemq: image: alfresco/alfresco-activemq:5.15.6 ports: diff --git a/search-services/packaging/src/docker/search_config_setup.sh b/search-services/packaging/src/docker/search_config_setup.sh index 40ad069e1..090a4c648 100644 --- a/search-services/packaging/src/docker/search_config_setup.sh +++ b/search-services/packaging/src/docker/search_config_setup.sh @@ -1,6 +1,47 @@ #!/bin/bash set -e +SOLR_CONFIG_FILE=$PWD/solrhome/templates/rerank/conf/solrconfig.xml +if [[ $REPLICATION_TYPE == "master" ]]; then + + findStringMaster='' + replaceStringMaster="\n\t \n" + if [[ $REPLICATION_AFTER == "" ]]; then + REPLICATION_AFTER=commit + fi + for i in $(echo $REPLICATION_AFTER | sed "s/,/ /g") + do + replaceStringMaster+="\t\t"$i"<\/str> \n" + done + if [[ ! -z "$REPLICATION_CONFIG_FILES" ]]; then + replaceStringMaster+="\t\t$REPLICATION_CONFIG_FILES<\/str> \n" + fi + replaceStringMaster+="\t<\/lst>" + sed -i -e "s/$findStringMaster/$findStringMaster$replaceStringMaster/g" $SOLR_CONFIG_FILE +fi +if [[ $REPLICATION_TYPE == "slave" ]]; then + if [[ $REPLICATION_MASTER_PROTOCOL == "" ]]; then + REPLICATION_MASTER_PROTOCOL=http + fi + if [[ $REPLICATION_MASTER_HOST == "" ]]; then + REPLICATION_MASTER_HOST=localhost + fi + if [[ $REPLICATION_MASTER_PORT == "" ]]; then + REPLICATION_MASTER_PORT=8083 + fi + if [[ $REPLICATION_CORE_NAME == "" ]]; then + REPLICATION_CORE_NAME=alfresco + fi + if [[ $REPLICATION_POLL_INTERVAL == "" ]]; then + REPLICATION_POLL_INTERVAL=00:00:30 + fi + sed -i 's//\ + \ + '$REPLICATION_MASTER_PROTOCOL':\/\/'$REPLICATION_MASTER_HOST':'$REPLICATION_MASTER_PORT'\/solr\/'$REPLICATION_CORE_NAME'<\/str>\ + '$REPLICATION_POLL_INTERVAL'<\/str>\ + <\/lst>/g' $SOLR_CONFIG_FILE +fi + SOLR_IN_FILE=$PWD/solr.in.sh if [[ ! -z "$MAX_SOLR_RAM_PERCENTAGE" ]]; then From 0fe58cb31fc2a145adaf8d35675d69040c35802b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2019 22:14:55 +0000 Subject: [PATCH 046/129] Bump restapi from 1.20 to 1.21 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.20 to 1.21. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.20...v1.21) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index e14adef45..a02127e07 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.20 + 1.21 1.11 3.0.16 3.2.0 From 19fe89ce17700fb8418f38f807d8653a5679626e Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Thu, 12 Dec 2019 11:41:57 +0000 Subject: [PATCH 047/129] SEARCH-2011 Fix Master ReplicationHandler in script --- search-services/packaging/src/docker/search_config_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/search_config_setup.sh b/search-services/packaging/src/docker/search_config_setup.sh index 090a4c648..a48589f0c 100644 --- a/search-services/packaging/src/docker/search_config_setup.sh +++ b/search-services/packaging/src/docker/search_config_setup.sh @@ -4,7 +4,7 @@ set -e SOLR_CONFIG_FILE=$PWD/solrhome/templates/rerank/conf/solrconfig.xml if [[ $REPLICATION_TYPE == "master" ]]; then - findStringMaster='' + findStringMaster='/' replaceStringMaster="\n\t \n" if [[ $REPLICATION_AFTER == "" ]]; then REPLICATION_AFTER=commit From 52cb47ce64141d1da3a3c5de5f1000397d1410a2 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 12 Dec 2019 15:49:49 +0100 Subject: [PATCH 048/129] [SEARCG-1994] - solved multiple version problem on windows. - fixed full replication not working on windows. --- .../java/org/alfresco/solr/content/SolrContentStore.java | 4 +++- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index a11f3f4ba..bc2dc600c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -43,6 +43,7 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; @@ -257,7 +258,8 @@ public final class SolrContentStore implements Closeable, AccessMode wr.write(Long.toString(version)); wr.close(); - tmpFile.renameTo(new File(root, ".version")); + Files.move(tmpFile.toPath(), new File(root, ".version").toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException exception) { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 66d676867..406736069 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1736,7 +1736,10 @@ class AlfrescoIndexFetcher private void cleanUpContentStore(String contentStorePath) throws Exception { AtomicInteger fileDeleted = new AtomicInteger(); - Set fileNames = contentStoreFilesToDownload.stream().map(e -> (String) e.get(NAME)) + Set fileNames = contentStoreFilesToDownload.stream() + .map(e -> (String) e.get(NAME)) + .map(Paths::get) + .map(p -> p.toString()) .collect(Collectors.toSet()); try { From 7567eb54c030b7f80975d0388cc6d927e976798e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2019 22:13:14 +0000 Subject: [PATCH 049/129] Bump restapi from 1.21 to 1.23 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.21 to 1.23. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.21...v1.23) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index a02127e07..a0509a394 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.21 + 1.23 1.11 3.0.16 3.2.0 From c91896e9e09fc1d326d986fa55ec7485185af328 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 13 Dec 2019 11:15:30 +0100 Subject: [PATCH 050/129] [SEARCH-1994] added comments --- .../java/org/alfresco/solr/content/SolrContentStore.java | 1 + .../org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index bc2dc600c..1233ce3c1 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -258,6 +258,7 @@ public final class SolrContentStore implements Closeable, AccessMode wr.write(Long.toString(version)); wr.close(); + // file.renameTo(..) does not work on windows. Use Files.move instead. Files.move(tmpFile.toPath(), new File(root, ".version").toPath(), StandardCopyOption.REPLACE_EXISTING); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 406736069..c3f450755 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1736,7 +1736,10 @@ class AlfrescoIndexFetcher private void cleanUpContentStore(String contentStorePath) throws Exception { AtomicInteger fileDeleted = new AtomicInteger(); - Set fileNames = contentStoreFilesToDownload.stream() + + // This is the set of the ONLY files that should be in contentStore. + // This set is computed from the information got from master and translated into the current OS path syntax. + Set contentStoreFiles = contentStoreFilesToDownload.stream() .map(e -> (String) e.get(NAME)) .map(Paths::get) .map(p -> p.toString()) @@ -1745,7 +1748,7 @@ class AlfrescoIndexFetcher { Files.walk(Paths.get(contentStorePath)).forEach(p -> { File f = new File(p.toUri()); - if (!f.isDirectory() && !fileNames.contains(p.toString().replace(contentStorePath, ""))) + if (!f.isDirectory() && !contentStoreFiles.contains(p.toString().replace(contentStorePath, ""))) { try { From 2f918ac8cdfddeb4f90cec9d4cc7d4365310d871 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 13 Dec 2019 11:17:10 +0100 Subject: [PATCH 051/129] [SEARCH-1994] replaced method in map --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index c3f450755..eacc0ed07 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1742,7 +1742,7 @@ class AlfrescoIndexFetcher Set contentStoreFiles = contentStoreFilesToDownload.stream() .map(e -> (String) e.get(NAME)) .map(Paths::get) - .map(p -> p.toString()) + .map(Path::toString) .collect(Collectors.toSet()); try { From 03f58e5af9ebb7474a1096c8b795e5001de651c3 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 13 Dec 2019 12:28:20 +0100 Subject: [PATCH 052/129] [SEARCH-1994] added comment --- .../java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index eacc0ed07..9b1407300 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1738,7 +1738,9 @@ class AlfrescoIndexFetcher AtomicInteger fileDeleted = new AtomicInteger(); // This is the set of the ONLY files that should be in contentStore. - // This set is computed from the information got from master and translated into the current OS path syntax. + // This set is computed from the information got from master. After a full replication, only the files + // that have been downloaded from master (contentStoreFilesToDownload) should be in contentStore. + // The file paths are translated in the current OS path notation. Set contentStoreFiles = contentStoreFilesToDownload.stream() .map(e -> (String) e.get(NAME)) .map(Paths::get) From a535688e8be512186c837cc6a2289ff332a8d074 Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Fri, 13 Dec 2019 11:38:28 +0000 Subject: [PATCH 053/129] SEARCH-2011 Update README about slave replica config setup --- README.md | 6 ++++++ search-services/packaging/src/docker/search_config_setup.sh | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b78fd2e7..43b13d542 100644 --- a/README.md +++ b/README.md @@ -41,3 +41,9 @@ More details are available at [search-services](/search-services) folder. **Following resources will not be available for Community users** More details are available at [insight-engine](/insight-engine) folder. + +### Enable Search Slave Replica config + +To enable slave node specify environment value `REPLICATION_TYPE=slave`, by default Master config is enabled and slave is disabled. + +During deployment time whenever Search Services or Insight Engine image starts, it will execute the script [search_config_setup.sh](/insight-engine/packaging/src/docker) which will configure the slave config setup based on the value specified in the script. diff --git a/search-services/packaging/src/docker/search_config_setup.sh b/search-services/packaging/src/docker/search_config_setup.sh index a48589f0c..fb7636649 100644 --- a/search-services/packaging/src/docker/search_config_setup.sh +++ b/search-services/packaging/src/docker/search_config_setup.sh @@ -1,5 +1,7 @@ #!/bin/bash set -e +# By default its going to deploy "Master" setup configuration with "REPLICATION_TYPE=master". +# Slave replica service can be enabled using "REPLICATION_TYPE=slave" environment value. SOLR_CONFIG_FILE=$PWD/solrhome/templates/rerank/conf/solrconfig.xml if [[ $REPLICATION_TYPE == "master" ]]; then @@ -17,7 +19,7 @@ if [[ $REPLICATION_TYPE == "master" ]]; then replaceStringMaster+="\t\t$REPLICATION_CONFIG_FILES<\/str> \n" fi replaceStringMaster+="\t<\/lst>" - sed -i -e "s/$findStringMaster/$findStringMaster$replaceStringMaster/g" $SOLR_CONFIG_FILE + sed -i "s/$findStringMaster/$findStringMaster$replaceStringMaster/g" $SOLR_CONFIG_FILE fi if [[ $REPLICATION_TYPE == "slave" ]]; then if [[ $REPLICATION_MASTER_PROTOCOL == "" ]]; then From 4880c0ad278b9a4cc07f37a3aff1313bd9550856 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 13 Dec 2019 13:45:35 +0100 Subject: [PATCH 054/129] [SEARCH-1994] use FilenameUtils.separetorToSystem to correct path in case of master slave on different OS(e.g. windows -> linux, linux->windows etc) --- .../alfresco/solr/handler/AlfrescoIndexFetcher.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 9b1407300..1b2e4f2c5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -37,6 +37,7 @@ package org.alfresco.solr.handler; import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.alfresco.solr.content.SolrContentStore; +import org.apache.commons.io.FilenameUtils; import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexCommit; @@ -188,6 +189,13 @@ class AlfrescoIndexFetcher private final Map confFileInfoCache = new HashMap<>(); private volatile Date replicationStartTimeStamp; private RTimer replicationTimer; + + /** + * The map contains the following fields: + * NAME : String -> file name(with path for contentstore files) + * SIZE : long -> file size + * CHECKSUM : long -> checksum + */ private volatile List> filesToDownload; private volatile List> confFilesToDownload; private volatile List> tlogFilesToDownload; @@ -197,6 +205,7 @@ class AlfrescoIndexFetcher private volatile List> confFilesDownloaded; private volatile List> tlogFilesDownloaded; private volatile List> contentStoreFilesDownloaded; + private volatile Map currentFile; private volatile DirectoryFileFetcher dirFileFetcher; private volatile LocalFsFileFetcher localFileFetcher; @@ -1743,8 +1752,7 @@ class AlfrescoIndexFetcher // The file paths are translated in the current OS path notation. Set contentStoreFiles = contentStoreFilesToDownload.stream() .map(e -> (String) e.get(NAME)) - .map(Paths::get) - .map(Path::toString) + .map(FilenameUtils::separatorsToSystem) .collect(Collectors.toSet()); try { From 2b83fbf7a6c2528016227a8d04fd8586419eb201 Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Mon, 16 Dec 2019 08:55:25 +0000 Subject: [PATCH 055/129] SEARCH-2011 Update README about slave replica config setup --- README.md | 8 +------- search-services/README.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 43b13d542..95ab30356 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,4 @@ More details are available at [search-services](/search-services) folder. **Following resources will not be available for Community users** -More details are available at [insight-engine](/insight-engine) folder. - -### Enable Search Slave Replica config - -To enable slave node specify environment value `REPLICATION_TYPE=slave`, by default Master config is enabled and slave is disabled. - -During deployment time whenever Search Services or Insight Engine image starts, it will execute the script [search_config_setup.sh](/insight-engine/packaging/src/docker) which will configure the slave config setup based on the value specified in the script. +More details are available at [insight-engine](/insight-engine) folder. \ No newline at end of file diff --git a/search-services/README.md b/search-services/README.md index f44e980ca..0c82e56d1 100644 --- a/search-services/README.md +++ b/search-services/README.md @@ -338,6 +338,22 @@ This Docker Image is available at Alfresco Docker Hub: To use the public image instead of the local one (`searchservices:develop`) just use `alfresco/alfresco-search-services:1.3.x.x` labels. +## Docker Master-Slave setup +### Enable Search Slave Replica config + +To enable slave node specify environment value `REPLICATION_TYPE=slave`, by default Master config is enabled and slave is disabled. + +During deployment time whenever Search Services or Insight Engine image starts, it will execute the script [search_config_setup.sh](/packaging/src/docker) which will configure the slave config setup based on the value specified in the script. + +To run the docker image: + +```bash +$ docker run -p 8984:8983 -e REPLICATION_TYPE=slave -e ALFRESCO_SECURE_COMMS=none -e SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive searchservices:develop +``` +Solr-slave End point: [http://localhost:8984/solr](http://localhost:8984/solr) + +To generate your own Docker-compose file please follow [generator-alfresco-docker-compose](../e2e-test/generator-alfresco-docker-compose/README.md) + ### Use Alfresco Search Services Docker Image with Docker Compose Sample configuration in a Docker Compose file using **Plain HTTP** protocol to communicate with Alfresco Repository. From c6f1c2a84d70e76a323d7b25f037e45e1e93022c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 22:12:36 +0000 Subject: [PATCH 056/129] Bump alfresco-data-model from 8.60 to 8.68 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.60 to 8.68. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.60...8.68) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index eb8c5dd03..983de5681 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.60 + 8.68 2.10.1 From f6ceaed333cfe1595c1ef7aab95d3fa7cbdef92a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 22:13:30 +0000 Subject: [PATCH 057/129] Bump slf4j.version from 1.7.29 to 1.7.30 in /search-services Bumps `slf4j.version` from 1.7.29 to 1.7.30. Updates `slf4j-log4j12` from 1.7.29 to 1.7.30 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.29...v_1.7.30) Updates `slf4j-api` from 1.7.29 to 1.7.30 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.29...v_1.7.30) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- search-services/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index eb8c5dd03..5067894da 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -73,7 +73,7 @@ org.slf4j slf4j-log4j12 - 1.7.29 + 1.7.30 diff --git a/search-services/pom.xml b/search-services/pom.xml index 364979b2c..9a316f644 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -14,7 +14,7 @@ pom Alfresco Solr Search parent - 1.7.29 + 1.7.30 3.2.5 From 1d95db37bca0d28610cf80b870469bd8f630f438 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 22:16:28 +0000 Subject: [PATCH 058/129] Bump mockito-core from 3.2.0 to 3.2.4 in /search-services Bumps [mockito-core](https://github.com/mockito/mockito) from 3.2.0 to 3.2.4. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.2.0...v3.2.4) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index a348e4fe5..34d4aed4c 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -158,7 +158,7 @@ org.mockito mockito-core - 3.2.0 + 3.2.4 test diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index eb8c5dd03..6ccb5c459 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -67,7 +67,7 @@ org.mockito mockito-core - 3.2.0 + 3.2.4 test From 3823e69d1d7eb5f8616cf1c58c83cb789005d54b Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 17 Dec 2019 08:32:28 +0000 Subject: [PATCH 059/129] Update license for XPP. --- .../{Apache-like-XPP.txt => BSDlike-XPP.txt} | 116 +++++++++--------- .../src/main/resources/licenses/notice.txt | 4 +- 2 files changed, 60 insertions(+), 60 deletions(-) rename search-services/packaging/src/main/resources/licenses/3rd-party/{Apache-like-XPP.txt => BSDlike-XPP.txt} (89%) diff --git a/search-services/packaging/src/main/resources/licenses/3rd-party/Apache-like-XPP.txt b/search-services/packaging/src/main/resources/licenses/3rd-party/BSDlike-XPP.txt similarity index 89% rename from search-services/packaging/src/main/resources/licenses/3rd-party/Apache-like-XPP.txt rename to search-services/packaging/src/main/resources/licenses/3rd-party/BSDlike-XPP.txt index fb11965fa..e4553dcaf 100644 --- a/search-services/packaging/src/main/resources/licenses/3rd-party/Apache-like-XPP.txt +++ b/search-services/packaging/src/main/resources/licenses/3rd-party/BSDlike-XPP.txt @@ -1,58 +1,58 @@ -LICENSE FOR THE Extreme! Lab PullParser ------------------------------------------------------------------------- - -Copyright © 2002 The Trustees of Indiana University. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1) All redistributions of source code must retain the above - copyright notice, the list of authors in the original source - code, this list of conditions and the disclaimer listed in this - license; - -2) All redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the disclaimer - listed in this license in the documentation and/or other - materials provided with the distribution; - -3) Any documentation included with all redistributions must include - the following acknowledgement: - - "This product includes software developed by the Indiana - University Extreme! Lab. For further information please visit - http://www.extreme.indiana.edu/" - - Alternatively, this acknowledgment may appear in the software - itself, and wherever such third-party acknowledgments normally - appear. - -4) The name "Indiana Univeristy" and "Indiana Univeristy - Extreme! Lab" shall not be used to endorse or promote - products derived from this software without prior written - permission from Indiana University. For written permission, - please contact http://www.extreme.indiana.edu/. - -5) Products derived from this software may not use "Indiana - Univeristy" name nor may "Indiana Univeristy" appear in their name, - without prior written permission of the Indiana University. - -Indiana University provides no reassurances that the source code -provided does not infringe the patent or any other intellectual -property rights of any other entity. Indiana University disclaims any -liability to any recipient for claims brought by any other entity -based on infringement of intellectual property rights or otherwise. - -LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH -NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. INDIANA -UNIVERSITY GIVES NO WARRANTIES AND MAKES NO REPRESENTATION THAT -SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR -OTHER PROPRIETARY RIGHTS.  INDIANA UNIVERSITY MAKES NO WARRANTIES THAT -SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP -DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE -RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS, -AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING -SOFTWARE. - +LICENSE FOR THE Extreme! Lab +------------------------------------------------------------------------ + +Copyright © 2003 The Trustees of Indiana University. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1) All redistributions of source code must retain the above + copyright notice, the list of authors in the original source + code, this list of conditions and the disclaimer listed in this + license; + +2) All redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the disclaimer + listed in this license in the documentation and/or other + materials provided with the distribution; + +3) Any documentation included with all redistributions must include + the following acknowledgement: + + "This product includes software developed by the Indiana + University Extreme! Lab. For further information please visit + http://www.extreme.indiana.edu/" + + Alternatively, this acknowledgment may appear in the software + itself, and wherever such third-party acknowledgments normally + appear. + +4) The name "Indiana University" and "Indiana University + Extreme! Lab" shall not be used to endorse or promote + products derived from this software without prior written + permission from Indiana University. For written permission, + please contact http://www.extreme.indiana.edu/. + +5) Products derived from this software may not use "Indiana + University" name nor may "Indiana University" appear in their name, + without prior written permission of the Indiana University. + +Indiana University provides no reassurances that the source code +provided does not infringe the patent or any other intellectual +property rights of any other entity. Indiana University disclaims any +liability to any recipient for claims brought by any other entity +based on infringement of intellectual property rights or otherwise. + +LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH +NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. INDIANA +UNIVERSITY GIVES NO WARRANTIES AND MAKES NO REPRESENTATION THAT +SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR +OTHER PROPRIETARY RIGHTS.  INDIANA UNIVERSITY MAKES NO WARRANTIES THAT +SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP +DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE +RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS, +AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING +SOFTWARE. + diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index ce669d559..80be1a313 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -22,8 +22,8 @@ antlr-3.5.2.jar http://www.antlr.org/ jaxen-1.2.0.jar http://www.cafeconleche.org/jaxen/ -=== Apache variant License === -xpp3-1.1.3_8.jar http://www.extreme.indiana.edu/xgws/xsoap/xpp/ +=== BSD variant License === +xpp3-1.1.4c.jar http://www.extreme.indiana.edu/dist/java-repository/xpp3/licenses/LICENSE.txt === JSON === From 687ce8096f4b6d8ef572716e797e0382f2ca340a Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 17 Dec 2019 08:35:18 +0000 Subject: [PATCH 060/129] Update quartz version in notice.txt. --- .../packaging/src/main/resources/licenses/notice.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 80be1a313..9932d0653 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -53,7 +53,7 @@ cxf-rt-wsdl-3.2.5.jar https://cxf.apache.org/ mybatis-spring-1.2.5.jar http://www.mybatis.org/ chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/ chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/ -quartz-2.3.1.jar http://quartz-scheduler.org/ +quartz-2.3.2.jar http://quartz-scheduler.org/ jackson-core-2.10.1.jar https://github.com/FasterXML/jackson jackson-annotations-2.10.1.jar https://github.com/FasterXML/jackson jackson-databind-2.10.1.jar https://github.com/FasterXML/jackson From 30fb35333f5cda23fd86037fc9ac3fb0cc1a083f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2019 12:54:45 +0000 Subject: [PATCH 061/129] Bump restapi from 1.23 to 1.25 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.23 to 1.25. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.23...v1.25) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index a0509a394..4dfc9b0ca 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.23 + 1.25 1.11 3.0.16 3.2.0 From f45dbaa85f3e5807c77a04886fb8298bba9ae3ff Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 17 Dec 2019 17:08:33 +0100 Subject: [PATCH 062/129] [SEARCH-1994] workaroung for open contentstore file descriptors in windows os --- .../java/org/alfresco/solr/content/SolrContentStore.java | 6 +++--- .../org/alfresco/solr/handler/AlfrescoIndexFetcher.java | 2 ++ .../alfresco/solr/handler/AlfrescoReplicationHandler.java | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index a11f3f4ba..bacb1d183 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.function.Predicate; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -234,10 +235,9 @@ public final class SolrContentStore implements Closeable, AccessMode @Override public long getLastCommittedVersion() { - try + try(Stream fileStream = Files.lines(Paths.get(root, ".version")) ) { - return Files.lines(Paths.get(root, ".version")) - .map(Long::parseLong) + return fileStream.map(Long::parseLong) .findFirst() .orElse(NO_VERSION_AVAILABLE); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 66d676867..e9546a2b0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -467,6 +467,8 @@ class AlfrescoIndexFetcher Map>> contentStoreMap = (Map>>) response .get(CONTENT_STORE_FILES); + fullContentStoreReplication = false; + if (contentStoreMap != null) { contentStoreFilesToDownload = Collections.synchronizedList(contentStoreMap.get(SolrContentStore.ADDS)); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java index be01b48b4..b382314e9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java @@ -2122,6 +2122,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So e.printStackTrace(); } } + } } catch (Exception e) @@ -2169,6 +2170,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So if (bytesRead <= 0) { writeNothingAndFlush(); + inputStream.close(); break; } @@ -2185,6 +2187,7 @@ public class AlfrescoReplicationHandler extends RequestHandlerBase implements So fos.flush(); } } + } } } From b1d842ba5d19640d8e5660e2320b20c874aa038c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2019 22:13:40 +0000 Subject: [PATCH 063/129] Bump utility from 3.0.16 to 3.0.17 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.16 to 3.0.17. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.16...utility-3.0.17) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index a0509a394..841c8c140 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 1.23 1.11 - 3.0.16 + 3.0.17 3.2.0 src/test/resources/SearchSuite.xml From 5a0b4bc1773e1e4f39c20e597a5a9938722ea8be Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Wed, 18 Dec 2019 10:17:24 +0100 Subject: [PATCH 064/129] [SEARCH-1994] fix tmp files moves --- .../solr/content/SolrContentStore.java | 19 +++++++++++++++---- .../solr/handler/AlfrescoIndexFetcher.java | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java index f08dcf288..1a3694a11 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java @@ -236,9 +236,11 @@ public final class SolrContentStore implements Closeable, AccessMode @Override public long getLastCommittedVersion() { - try(Stream fileStream = Files.lines(Paths.get(root, ".version")) ) + try { - return fileStream.map(Long::parseLong) + return Files.readAllLines(Paths.get(root, ".version")) + .stream() + .map(Long::parseLong) .findFirst() .orElse(NO_VERSION_AVAILABLE); } @@ -251,20 +253,29 @@ public final class SolrContentStore implements Closeable, AccessMode @Override public void setLastCommittedVersion(long version) { + + File tmpFile = new File(root, ".version-" + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date())); try { - File tmpFile = new File(root, ".version-" + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date())); FileWriter wr = new FileWriter(tmpFile); wr.write(Long.toString(version)); wr.close(); // file.renameTo(..) does not work on windows. Use Files.move instead. - Files.move(tmpFile.toPath(), new File(root, ".version").toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move(tmpFile.toPath(), new File(root, ".version").toPath(), StandardCopyOption.ATOMIC_MOVE); } catch (IOException exception) { logger.error("Unable to persist the last committed content store version {}. See the stacktrace below for furtger details.", version, exception); + try + { + Files.delete(tmpFile.toPath()); + } + catch (IOException e) + { + logger.error("Unable to delete tmp contentstore version file {}.", version); + } } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java index 536ce0354..bb36889e7 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java @@ -1714,7 +1714,7 @@ class AlfrescoIndexFetcher try { Files.createDirectories(Paths.get(csFile.getParent())); - Files.copy(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move(tmpFile.toPath(), csFile.toPath(), StandardCopyOption.ATOMIC_MOVE); } catch (IOException e) { From 2744500fed6d0729e2cf6140122aa11f65a3c3b2 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 18 Dec 2019 10:34:23 +0000 Subject: [PATCH 065/129] SEARCH-2012 Minor refactoring of FacetRangeSearchTest. --- .../search/FacetRangeSearchTest.java | 268 ++++++++---------- 1 file changed, 121 insertions(+), 147 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java index 4dfc0c2b9..22dbc169c 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java @@ -22,9 +22,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -44,100 +42,83 @@ import org.testng.annotations.Test; /** * Faceted Range Search Query for numeric range * { - * "query": { - * "query": "name:A*" - * }, - * "range": { - * "field": "content.size", - * "start": "0", - * "end": "400", - * "gap": "100" - * } + * "query": { + * "query": "name:A*" + * }, + * "range": { + * "field": "content.size", + * "start": "0", + * "end": "400", + * "gap": "100" + * } * } * Date range query: * { - * "query": { - * "query": "name:A*" - * }, - * "range": { - * "field": "created", - * "start": "2015-09-29T10:45:15.729Z", - * "end": "2016-09-29T10:45:15.729Z", - * "gap": "+100DAY" - * } + * "query": { + * "query": "name:A*" + * }, + * "range": { + * "field": "created", + * "start": "2015-09-29T10:45:15.729Z", + * "end": "2016-09-29T10:45:15.729Z", + * "gap": "+100DAY" + * } * } - * @author Michael Suzuki * + * @author Michael Suzuki */ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest { - @BeforeClass(alwaysRun = true) + @BeforeClass (alwaysRun = true) public void dataPreparation() throws Exception { searchServicesDataPreparation(); waitForContentIndexing(file4.getContent(), true); } + /** Check the error messages mention the mandatory fields when they are omitted. */ @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check facet intervals mandatory fields") public void checkingFacetsMandatoryErrorMessages() { SearchRequest query = createQuery("cars"); - List ranges = new ArrayList<>(); - RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); - ranges.add(facetRangeModel); - query.setRanges(ranges); + + // Omit the field. + query.setRanges(List.of(createRangesModel(null, "0", "400", "20"))); query(query); - restClient.assertStatusCodeIs(HttpStatus.BAD_REQUEST).assertLastError() - .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "field")); - ranges.clear(); - facetRangeModel.setField("content.size"); - ranges.add(facetRangeModel); - query.setRanges(ranges); + .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "field")); + // Omit the start. + query.setRanges(List.of(createRangesModel("content.size", null, "400", "20"))); query(query); - restClient.assertStatusCodeIs(HttpStatus.BAD_REQUEST).assertLastError() - .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "start")); - facetRangeModel.setStart("0"); - ranges.clear(); - ranges.add(facetRangeModel); - query.setRanges(ranges); + .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "start")); + // Omit the end. + query.setRanges(List.of(createRangesModel("content.size", "0", null, "20"))); query(query); - restClient.assertStatusCodeIs(HttpStatus.BAD_REQUEST).assertLastError() - .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "end")); - facetRangeModel.setEnd("400"); - query.setRanges(ranges); - ranges.clear(); - ranges.add(facetRangeModel); + .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "end")); + // Omit the gap. + query.setRanges(List.of(createRangesModel("content.size", "0", "400", null))); query(query); - restClient.assertStatusCodeIs(HttpStatus.BAD_REQUEST).assertLastError() - .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "gap")); - - facetRangeModel.setGap("100"); + .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "gap")); } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") - @SuppressWarnings("unchecked") + @SuppressWarnings ("unchecked") public void searchWithRange() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); - RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); - facetRangeModel.setField("content.size"); - facetRangeModel.setStart("0"); - facetRangeModel.setEnd("200"); - facetRangeModel.setGap("20"); - List ranges = new ArrayList<>(); - ranges.add(facetRangeModel); + RestRequestRangesModel facetRangeModel = createRangesModel("content.size", "0", "200", "20"); + List ranges = List.of(facetRangeModel); query.setRanges(ranges); SearchResponse response = query(query); response.assertThat().entriesListIsNotEmpty(); @@ -147,14 +128,14 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest RestGenericBucketModel bucket = facetResponseModel.getBuckets().get(0); bucket.assertThat().field("label").is("[20 - 40)"); bucket.assertThat().field("filterQuery").is("content.size:[\"20\" TO \"40\">"); - Map metric = (Map) bucket.getMetrics().get(0).getValue(); + Map metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 2, "Unexpected count for first bucket."); Map info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"20"); - assertEquals(info.get("end"),"40"); + assertEquals(info.get("start"), "20"); + assertEquals(info.get("end"), "40"); assertNull(info.get("count")); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"false"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "false"); bucket = facetResponseModel.getBuckets().get(1); bucket.assertThat().field("label").is("[40 - 120)"); @@ -162,38 +143,33 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 1, "Unexpected count for second bucket."); info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"40"); - assertEquals(info.get("end"),"120"); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"false"); + assertEquals(info.get("start"), "40"); + assertEquals(info.get("end"), "120"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "false"); bucket = facetResponseModel.getBuckets().get(2); bucket.assertThat().field("label").is("[120 - 200]"); bucket.assertThat().field("filterQuery").is("content.size:[\"120\" TO \"200\"]"); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 1, "Unexpected count for third bucket."); info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"120"); - assertEquals(info.get("end"),"200"); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"true"); + assertEquals(info.get("start"), "120"); + assertEquals(info.get("end"), "200"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "true"); } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") - @SuppressWarnings("unchecked") + @SuppressWarnings ("unchecked") public void searchWithRangeHardend() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); - RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); - facetRangeModel.setField("content.size"); - facetRangeModel.setStart("0"); - facetRangeModel.setEnd("200"); - facetRangeModel.setGap("20"); + RestRequestRangesModel facetRangeModel = createRangesModel("content.size", "0", "200", "20"); facetRangeModel.setHardend(true); - List ranges = new ArrayList<>(); - ranges.add(facetRangeModel); + List ranges = List.of(facetRangeModel); query.setRanges(ranges); SearchResponse response = query(query); response.assertThat().entriesListIsNotEmpty(); @@ -203,26 +179,26 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest RestGenericBucketModel bucket = facetResponseModel.getBuckets().get(0); bucket.assertThat().field("label").is("[20 - 40)"); bucket.assertThat().field("filterQuery").is("content.size:[\"20\" TO \"40\">"); - Map metric = (Map) bucket.getMetrics().get(0).getValue(); + Map metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 2, "Unexpected count for first bucket."); Map info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"20"); - assertEquals(info.get("end"),"40"); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"false"); + assertEquals(info.get("start"), "20"); + assertEquals(info.get("end"), "40"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "false"); assertNull(info.get("count")); bucket = facetResponseModel.getBuckets().get(1); bucket.assertThat().field("label").is("[40 - 120)"); bucket.assertThat().field("filterQuery").is("content.size:[\"40\" TO \"120\">"); info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"40"); - assertEquals(info.get("end"),"120"); + assertEquals(info.get("start"), "40"); + assertEquals(info.get("end"), "120"); metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 1, "Unexpected count for second bucket."); assertNull(info.get("count")); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"false"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "false"); bucket = facetResponseModel.getBuckets().get(2); bucket.assertThat().field("label").is("[120 - 200]"); @@ -230,29 +206,24 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 1, "Unexpected count for third bucket."); info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"120"); - assertEquals(info.get("end"),"200"); + assertEquals(info.get("start"), "120"); + assertEquals(info.get("end"), "200"); assertNull(info.get("count")); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"true"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "true"); } /** This test relies on a document created in 2015 existing, probably part of the sample site. */ @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") - @SuppressWarnings("unchecked") + @SuppressWarnings ("unchecked") public void searchDateRange() { SearchRequest query = createQuery("name:A*"); - RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); - facetRangeModel.setField("created"); - facetRangeModel.setStart("2015-09-29T10:45:15.729Z"); - facetRangeModel.setEnd("2016-09-29T10:45:15.729Z"); - facetRangeModel.setGap("+280DAY"); - List ranges = new ArrayList<>(); - ranges.add(facetRangeModel); + RestRequestRangesModel facetRangeModel = createRangesModel("created", "2015-09-29T10:45:15.729Z", "2016-09-29T10:45:15.729Z", "+280DAY"); + List ranges = List.of(facetRangeModel); query.setRanges(ranges); SearchResponse response = query(query); response.assertThat().entriesListIsNotEmpty(); @@ -260,60 +231,44 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest RestGenericFacetResponseModel facetResponseModel = response.getContext().getFacets().get(0); List buckets = facetResponseModel.getBuckets(); - assertThat(buckets.size(),is(1)); + assertThat(buckets.size(), is(1)); RestGenericBucketModel bucket = buckets.get(0); bucket.assertThat().field("label").is("[2015-09-29T10:45:15.729Z - 2017-04-11T10:45:15.729Z]"); bucket.assertThat().field("filterQuery").is("created:[\"2015-09-29T10:45:15.729Z\" TO \"2017-04-11T10:45:15.729Z\"]"); bucket.getMetrics().get(0).assertThat().field("value").is("{count=1}"); Map info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"2015-09-29T10:45:15.729Z"); - assertEquals(info.get("end"),"2017-04-11T10:45:15.729Z"); - assertNull(info.get("count"),"1"); - assertEquals(info.get("startInclusive"),"true"); - assertEquals(info.get("endInclusive"),"true"); + assertEquals(info.get("start"), "2015-09-29T10:45:15.729Z"); + assertEquals(info.get("end"), "2017-04-11T10:45:15.729Z"); + assertNull(info.get("count"), "1"); + assertEquals(info.get("startInclusive"), "true"); + assertEquals(info.get("endInclusive"), "true"); } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") public void searchDateAndSizeRanges() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); - List ranges = new ArrayList<>(); - RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); - facetRangeModel.setField("created"); - facetRangeModel.setStart("2015-09-29T10:45:15.729Z"); - facetRangeModel.setEnd("2016-09-29T10:45:15.729Z"); - facetRangeModel.setGap("+280DAY"); - ranges.add(facetRangeModel); - RestRequestRangesModel facetCountRangeModel = new RestRequestRangesModel(); - facetCountRangeModel.setField("content.size"); - facetCountRangeModel.setStart("0"); - facetCountRangeModel.setEnd("500"); - facetCountRangeModel.setGap("200"); - ranges.add(facetCountRangeModel); + RestRequestRangesModel facetRangeModel = createRangesModel("created", "2015-09-29T10:45:15.729Z", "2016-09-29T10:45:15.729Z", "+280DAY"); + RestRequestRangesModel facetCountRangeModel = createRangesModel("content.size", "0", "500", "200"); + List ranges = List.of(facetRangeModel, facetCountRangeModel); query.setRanges(ranges); } @Test - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, + @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") - @SuppressWarnings("unchecked") + @SuppressWarnings ("unchecked") public void searchWithRangeAndIncludeUpperBound() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); - RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); - facetRangeModel.setField("content.size"); - facetRangeModel.setStart("0"); - facetRangeModel.setEnd("200"); - facetRangeModel.setGap("20"); - List include = new ArrayList<>(); - include.add("upper"); + RestRequestRangesModel facetRangeModel = createRangesModel("content.size", "0", "200", "20"); + List include = List.of("upper"); facetRangeModel.setInclude(include); - List ranges = new ArrayList<>(); - ranges.add(facetRangeModel); + List ranges = List.of(facetRangeModel); query.setRanges(ranges); SearchResponse response = query(query); response.assertThat().entriesListIsNotEmpty(); @@ -323,14 +278,14 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest RestGenericBucketModel bucket = facetResponseModel.getBuckets().get(0); bucket.assertThat().field("label").is("(20 - 40]"); bucket.assertThat().field("filterQuery").is("content.size:<\"20\" TO \"40\"]"); - Map metric = (Map) bucket.getMetrics().get(0).getValue(); + Map metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 2, "Unexpected count for first bucket."); Map info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"20"); - assertEquals(info.get("end"),"40"); + assertEquals(info.get("start"), "20"); + assertEquals(info.get("end"), "40"); assertNull(info.get("count")); - assertEquals(info.get("startInclusive"),"false"); - assertEquals(info.get("endInclusive"),"true"); + assertEquals(info.get("startInclusive"), "false"); + assertEquals(info.get("endInclusive"), "true"); bucket = facetResponseModel.getBuckets().get(1); bucket.assertThat().field("label").is("(40 - 120]"); @@ -338,10 +293,10 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 1, "Unexpected count for second bucket."); info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"40"); - assertEquals(info.get("end"),"120"); - assertEquals(info.get("startInclusive"),"false"); - assertEquals(info.get("endInclusive"),"true"); + assertEquals(info.get("start"), "40"); + assertEquals(info.get("end"), "120"); + assertEquals(info.get("startInclusive"), "false"); + assertEquals(info.get("endInclusive"), "true"); bucket = facetResponseModel.getBuckets().get(2); bucket.assertThat().field("label").is("(120 - 200]"); @@ -349,9 +304,28 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest metric = (Map) bucket.getMetrics().get(0).getValue(); assertEquals(Integer.valueOf(metric.get("count")).intValue(), 1, "Unexpected count for third bucket."); info = (Map) bucket.getBucketInfo(); - assertEquals(info.get("start"),"120"); - assertEquals(info.get("end"),"200"); - assertEquals(info.get("startInclusive"),"false"); - assertEquals(info.get("endInclusive"),"true"); - } + assertEquals(info.get("start"), "120"); + assertEquals(info.get("end"), "200"); + assertEquals(info.get("startInclusive"), "false"); + assertEquals(info.get("endInclusive"), "true"); + } + + /** + * Create a ranges model with the values given. + * + * @param field The field to facet on. + * @param start The lowest facet value. + * @param end The highest facet value. + * @param gap The size of the buckets. + * @return The facet ranges model. + */ + private RestRequestRangesModel createRangesModel(String field, String start, String end, String gap) + { + RestRequestRangesModel facetRangeModel = new RestRequestRangesModel(); + facetRangeModel.setField(field); + facetRangeModel.setStart(start); + facetRangeModel.setEnd(end); + facetRangeModel.setGap(gap); + return facetRangeModel; + } } \ No newline at end of file From 2e62e3736b5c6f8d732884d5004c2f1e964877f5 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 18 Dec 2019 11:51:57 +0000 Subject: [PATCH 066/129] SEARCH-2012 Undo unwanted whitespace changes. --- .../search/FacetRangeSearchTest.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java index 22dbc169c..5b52698a2 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java @@ -42,34 +42,34 @@ import org.testng.annotations.Test; /** * Faceted Range Search Query for numeric range * { - * "query": { - * "query": "name:A*" - * }, - * "range": { - * "field": "content.size", - * "start": "0", - * "end": "400", - * "gap": "100" - * } + * "query": { + * "query": "name:A*" + * }, + * "range": { + * "field": "content.size", + * "start": "0", + * "end": "400", + * "gap": "100" + * } * } * Date range query: * { - * "query": { - * "query": "name:A*" - * }, - * "range": { - * "field": "created", - * "start": "2015-09-29T10:45:15.729Z", - * "end": "2016-09-29T10:45:15.729Z", - * "gap": "+100DAY" - * } + * "query": { + * "query": "name:A*" + * }, + * "range": { + * "field": "created", + * "start": "2015-09-29T10:45:15.729Z", + * "end": "2016-09-29T10:45:15.729Z", + * "gap": "+100DAY" + * } * } * * @author Michael Suzuki */ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest { - @BeforeClass (alwaysRun = true) + @BeforeClass(alwaysRun = true) public void dataPreparation() throws Exception { searchServicesDataPreparation(); @@ -78,7 +78,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest /** Check the error messages mention the mandatory fields when they are omitted. */ @Test - @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check facet intervals mandatory fields") public void checkingFacetsMandatoryErrorMessages() { @@ -110,9 +110,9 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") - @SuppressWarnings ("unchecked") + @SuppressWarnings("unchecked") public void searchWithRange() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); @@ -160,9 +160,9 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") - @SuppressWarnings ("unchecked") + @SuppressWarnings("unchecked") public void searchWithRangeHardend() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); @@ -215,9 +215,9 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest /** This test relies on a document created in 2015 existing, probably part of the sample site. */ @Test - @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") - @SuppressWarnings ("unchecked") + @SuppressWarnings("unchecked") public void searchDateRange() { SearchRequest query = createQuery("name:A*"); @@ -246,7 +246,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") public void searchDateAndSizeRanges() { @@ -258,9 +258,9 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest } @Test - @TestRail (section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") - @SuppressWarnings ("unchecked") + @SuppressWarnings("unchecked") public void searchWithRangeAndIncludeUpperBound() { SearchRequest query = createQuery("* AND SITE:'" + testSite.getId() + "'"); From 3f46073f93e324b1c84c264ea0c4f0818cb2150f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2019 22:14:35 +0000 Subject: [PATCH 067/129] Bump alfresco-data-model from 8.68 to 8.70 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.68 to 8.70. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.68...8.70) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index bd59eb1c3..82d0d83ff 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.68 + 8.70 2.10.1 From 89f92e960ed05df68554ed864211796a00c83316 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 19 Dec 2019 12:13:45 +0000 Subject: [PATCH 068/129] Merge branch 'feature/SEARCH-2012_NullDocsWithField_14x' into 'release/V1.4.x' Feature/search 2012 null docs with field 14x See merge request search_discovery/insightengine!280 (cherry picked from commit 680eae4b6aaa8e04cc280af0c2057f6a2d868c2a) --- .../AlfrescoCollatableMLTextFieldType.java | 49 +++-- .../solr/AlfrescoCollatableTextFieldType.java | 18 +- ...AlfrescoCollatableMLTextFieldTypeTest.java | 200 ++++++++++++++++++ .../AlfrescoCollatableTextFieldTypeTest.java | 168 +++++++++++++++ .../solr/query/AlfrescoSolrSortIT.java | 108 ++++++++++ 5 files changed, 513 insertions(+), 30 deletions(-) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldTypeTest.java create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableTextFieldTypeTest.java create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSortIT.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldType.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldType.java index 03d31c672..89d171642 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldType.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldType.java @@ -39,12 +39,9 @@ import org.springframework.extensions.surf.util.I18NUtil; /** * @author Andy - * */ public class AlfrescoCollatableMLTextFieldType extends StrField { - - /* (non-Javadoc) * @see org.apache.solr.schema.StrField#getSortField(org.apache.solr.schema.SchemaField, boolean) */ @@ -75,7 +72,6 @@ public class AlfrescoCollatableMLTextFieldType extends StrField } - public static class MLTextSortFieldComparatorSource extends FieldComparatorSource { @@ -101,16 +97,20 @@ public class AlfrescoCollatableMLTextFieldType extends StrField private final String[] values; private BinaryDocValues docTerms; - - private Bits docsWithField; + + /** + * An array of flags - one for each document in the segment. Each bit is set to true if the document has the + * field or false otherwise. If this is set to null then all docs in the segment have the field. + */ + Bits docsWithField; private final String field; - final Collator collator; + Collator collator; - private String bottom; - - private String top; + String bottom; + + String top; Locale collatorLocale; @@ -138,7 +138,7 @@ public class AlfrescoCollatableMLTextFieldType extends StrField { final String comparableString = findBestValue(doc, docTerms.get(doc)); return compareValues(bottom, comparableString); - + } public void copy(int slot, int doc) @@ -153,13 +153,14 @@ public class AlfrescoCollatableMLTextFieldType extends StrField private String findBestValue(int doc, BytesRef term) { - if (term.length == 0 && docsWithField.get(doc) == false) { + if (term.length == 0 && docsWithField != null && docsWithField.get(doc) == false) + { return null; } - + String withLocale = term.utf8ToString(); - - // split strin into MLText object + + // split string into MLText object if (withLocale == null) { return withLocale; @@ -231,14 +232,15 @@ public class AlfrescoCollatableMLTextFieldType extends StrField { docTerms = DocValues.getBinary(context.reader(), field); docsWithField = DocValues.getDocsWithField(context.reader(), field); - if (docsWithField instanceof Bits.MatchAllBits) { - docsWithField = null; + if (docsWithField instanceof Bits.MatchAllBits) + { + docsWithField = null; } return this; } - + @Override - public int compareValues(String val1, String val2) + public int compareValues(String val1, String val2) { if (val1 == null) { @@ -254,9 +256,10 @@ public class AlfrescoCollatableMLTextFieldType extends StrField } return collator.compare(val1, val2); } - - @Override - public void setScorer(Scorer scorer) {} - } + @Override + public void setScorer(Scorer scorer) + { + } + } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableTextFieldType.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableTextFieldType.java index d3d54cd63..604dbcc27 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableTextFieldType.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableTextFieldType.java @@ -104,16 +104,20 @@ public class AlfrescoCollatableTextFieldType extends StrField private final String[] values; private BinaryDocValues docTerms; - - private Bits docsWithField; + + /** + * An array of flags - one for each document in the segment. Each bit is set to true if the document has the + * field or false otherwise. If this is set to null then all docs in the segment have the field. + */ + Bits docsWithField; private final String field; - final Collator collator; + Collator collator; - private String bottom; + String bottom; - private String top; + String top; Locale collatorLocale; @@ -141,7 +145,6 @@ public class AlfrescoCollatableTextFieldType extends StrField { final String comparableString = findBestValue(doc, docTerms.get(doc)); return compareValues(bottom, comparableString); - } public void copy(int slot, int doc) @@ -156,7 +159,8 @@ public class AlfrescoCollatableTextFieldType extends StrField private String findBestValue(int doc, BytesRef term) { - if (term.length == 0 && docsWithField.get(doc) == false) { + if (term.length == 0 && docsWithField != null && docsWithField.get(doc) == false) + { return null; } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldTypeTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldTypeTest.java new file mode 100644 index 000000000..dcd57f47e --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableMLTextFieldTypeTest.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.text.Collator; +import java.util.Locale; + +import org.alfresco.solr.AlfrescoCollatableMLTextFieldType.MLTextSortFieldComparator; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; + +/** Unit tests for {@link AlfrescoCollatableMLTextFieldType}. */ +public class AlfrescoCollatableMLTextFieldTypeTest +{ + private static final int NUM_HITS = 3; + private static final String FIELD = "field"; + private static final Locale LOCALE = Locale.getDefault(); + /** A document id. */ + private static final int DOC = 0; + /** A value for the current bottom document. */ + private static final String BOTTOM_STRING = "Bottom"; + + @InjectMocks + MLTextSortFieldComparator textSortFieldComparator = new MLTextSortFieldComparator(NUM_HITS, FIELD, LOCALE); + @Mock + BinaryDocValues mockDocTerms; + @Mock + Bits mockDocsWithField; + @Mock + Collator mockCollator; + + @Before + public void setUp() + { + initMocks(this); + reset(mockDocTerms, mockDocsWithField); + textSortFieldComparator.bottom = BOTTOM_STRING; + } + + /** Check that a zero length term is sorted before a populated field. */ + @Test + public void testCompareBottom_termLengthZeroAndDocDoesntHaveField() + { + // Set up the document to have an empty term. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef()); + when(mockDocsWithField.get(DOC)).thenReturn(false); + + // Call the method under test. + int result = textSortFieldComparator.compareBottom(DOC); + + assertEquals("Expected value for doc to be null, and so it to be sorted before BOTTOM_TERM", 1, result); + } + + /** + * Check the behaviour of compareBottom when docsWithField is null (this happens when all documents contain the + * field). + */ + @Test + public void testCompareBottom_nullDocsWithField() + { + // Set docsWithField to null to simulate all documents containing the field. + Bits oldValue = textSortFieldComparator.docsWithField; + textSortFieldComparator.docsWithField = null; + + // Set up the document to have an empty term. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef()); + + // Call the method under test. + textSortFieldComparator.compareBottom(DOC); + + // Expect the EMPTY_TERM to be compared + verify(mockCollator).compare(BOTTOM_STRING, ""); + + // Reset docsWithField with the mock after the test. + textSortFieldComparator.docsWithField = oldValue; + } + + /** Check that if the doc has a value then it is compared with the existing value. */ + @Test + public void testCompareBottom_populatedTerm() + { + // Set up the document to have "Some value" for the field. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef("Some value")); + when(mockDocsWithField.get(DOC)).thenReturn(false); + + // Call the method under test. + textSortFieldComparator.compareBottom(DOC); + + verify(mockCollator).compare(BOTTOM_STRING, "Some value"); + } + + /** Check the behaviour if the multilanguage term is encoded. */ + @Test + public void testCompareBottom_encodedTerm_localeFound() + { + // Create an encoded multilanguage string with Russian, US English and Thai with Thai digits. + String mlText = "\u0000ru\u0000First\u0000Ignored" + + "\u0000en_US\u0000Second\u0000IgnoredToo" + + "\u0000th_TH_TH\u0000Third\u0000AlsoIgnored"; + // Set up the document to have an encoded value for the field. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef(mlText)); + when(mockDocsWithField.get(DOC)).thenReturn(false); + + // Check that the Russian text can be extracted. + textSortFieldComparator.collatorLocale = Locale.forLanguageTag("ru"); + textSortFieldComparator.compareBottom(DOC); + verify(mockCollator).compare(BOTTOM_STRING, "First"); + + // Check that the English text can be extracted. + textSortFieldComparator.collatorLocale = Locale.forLanguageTag("en"); + textSortFieldComparator.compareBottom(DOC); + verify(mockCollator).compare(BOTTOM_STRING, "Second"); + + // Check that the Thai text can be extracted. + textSortFieldComparator.collatorLocale = Locale.forLanguageTag("th"); + textSortFieldComparator.compareBottom(DOC); + verify(mockCollator).compare(BOTTOM_STRING, "Third"); + + // Reset the locale for other tests. + textSortFieldComparator.collatorLocale = LOCALE; + } + + /** Check the behaviour if the term has a locale but no text. */ + @Test + public void testCompareBottom_badlyEncodedTerm() + { + // Set the value to have a locale but no text. + String mlText = "\u0000ru"; + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef(mlText)); + + // Call the method under test. + textSortFieldComparator.compareBottom(DOC); + + // Check that an empty string is assumed. + verify(mockCollator).compare(BOTTOM_STRING, ""); + } + + @Test + public void testCompareValues_nullLessThanString() + { + int result = textSortFieldComparator.compareValues(null, "NotNull"); + assertEquals("Expected null to be 'less' than string.", -1, result); + } + + @Test + public void testCompareValues_stringGreaterThanNull() + { + int result = textSortFieldComparator.compareValues("NotNull", null); + assertEquals("Expected string to be 'greater' than null.", 1, result); + } + + @Test + public void testCompareValues_nullEqualToNull() + { + int result = textSortFieldComparator.compareValues(null, null); + assertEquals("Expected two null values to be equal.", 0, result); + } + + /** Check that when two non-null strings are compared then the underlying collator is used to get the result. */ + @Test + public void testCompareValues_twoStringsCompared() + { + // An arbitrary value to be returned by the collator. + int comparisonResult = 10; + when(mockCollator.compare("NotNull1", "NotNull2")).thenReturn(comparisonResult); + + // Call the method under test. + int result = textSortFieldComparator.compareValues("NotNull1", "NotNull2"); + + verify(mockCollator).compare("NotNull1", "NotNull2"); + assertEquals("Expected result to be obtained from collator.", comparisonResult, result); + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableTextFieldTypeTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableTextFieldTypeTest.java new file mode 100644 index 000000000..be8609c27 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCollatableTextFieldTypeTest.java @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.text.Collator; +import java.util.Locale; + +import org.alfresco.solr.AlfrescoCollatableTextFieldType.TextSortFieldComparator; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; + +/** Unit tests for {@link AlfrescoCollatableTextFieldType}. */ +public class AlfrescoCollatableTextFieldTypeTest +{ + private static final int NUM_HITS = 3; + private static final String FIELD = "field"; + private static final Locale LOCALE = Locale.getDefault(); + /** A document id. */ + private static final int DOC = 0; + /** A value for the current bottom document. */ + private static final String BOTTOM_STRING = "Bottom"; + + @InjectMocks + TextSortFieldComparator textSortFieldComparator = new TextSortFieldComparator(NUM_HITS, FIELD, LOCALE); + @Mock + BinaryDocValues mockDocTerms; + @Mock + Bits mockDocsWithField; + @Mock + Collator mockCollator; + + @Before + public void setUp() + { + initMocks(this); + reset(mockDocTerms, mockDocsWithField); + textSortFieldComparator.bottom = BOTTOM_STRING; + } + + /** Check that a zero length term is sorted before a populated field. */ + @Test + public void testCompareBottom_termLengthZeroAndDocDoesntHaveField() + { + // Set up the document to have an empty term. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef()); + when(mockDocsWithField.get(DOC)).thenReturn(false); + + // Call the method under test. + int result = textSortFieldComparator.compareBottom(DOC); + + assertEquals("Expected value for doc to be null, and so it to be sorted before BOTTOM_TERM", 1, result); + } + + /** + * Check the behaviour of compareBottom when docsWithField is null (this happens when all documents contain the + * field). + */ + @Test + public void testCompareBottom_nullDocsWithField() + { + // Set docsWithField to null to simulate all documents containing the field. + Bits oldValue = textSortFieldComparator.docsWithField; + textSortFieldComparator.docsWithField = null; + + // Set up the document to have an empty term. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef()); + + // Call the method under test. + textSortFieldComparator.compareBottom(DOC); + + // Expect the EMPTY_TERM to be compared + verify(mockCollator).compare(BOTTOM_STRING, ""); + + // Reset docsWithField with the mock after the test. + textSortFieldComparator.docsWithField = oldValue; + } + + /** Check that if the doc has a value then it is compared with the existing value. */ + @Test + public void testCompareBottom_populatedTerm() + { + // Set up the document to have "Some value" for the field. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef("Some value")); + when(mockDocsWithField.get(DOC)).thenReturn(false); + + // Call the method under test. + textSortFieldComparator.compareBottom(DOC); + + verify(mockCollator).compare(BOTTOM_STRING, "Some value"); + } + + /** Check the behaviour if the term is encoded. */ + @Test + public void testCompareBottom_encodedTerm() + { + // Set up the document to have an encoded value for the field. + when(mockDocTerms.get(DOC)).thenReturn(new BytesRef("\u0000Value\u0000Ignored")); + when(mockDocsWithField.get(DOC)).thenReturn(false); + + // Call the method under test. + textSortFieldComparator.compareBottom(DOC); + + verify(mockCollator).compare(BOTTOM_STRING, "Value"); + } + + @Test + public void testCompareValues_nullLessThanString() + { + int result = textSortFieldComparator.compareValues(null, "NotNull"); + assertEquals("Expected null to be 'less' than string.", -1, result); + } + + @Test + public void testCompareValues_stringGreaterThanNull() + { + int result = textSortFieldComparator.compareValues("NotNull", null); + assertEquals("Expected string to be 'greater' than null.", 1, result); + } + + @Test + public void testCompareValues_nullEqualToNull() + { + int result = textSortFieldComparator.compareValues(null, null); + assertEquals("Expected two null values to be equal.", 0, result); + } + + /** Check that when two non-null strings are compared then the underlying collator is used to get the result. */ + @Test + public void testCompareValues_twoStringsCompared() + { + // An arbitrary value to be returned by the collator. + int comparisonResult = 10; + when(mockCollator.compare("NotNull1", "NotNull2")).thenReturn(comparisonResult); + + // Call the method under test. + int result = textSortFieldComparator.compareValues("NotNull1", "NotNull2"); + + verify(mockCollator).compare("NotNull1", "NotNull2"); + assertEquals("Expected result to be obtained from collator.", comparisonResult, result); + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSortIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSortIT.java new file mode 100644 index 000000000..dd269a25b --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/AlfrescoSolrSortIT.java @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.query; + +import org.alfresco.solr.AbstractAlfrescoDistributedIT; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; + +/** + * https://issues.alfresco.com/jira/browse/SEARCH-2012 + */ +@SolrTestCaseJ4.SuppressSSL +@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) +public class AlfrescoSolrSortIT extends AbstractAlfrescoDistributedIT +{ + @BeforeClass + private static void initData() throws Throwable + { + initSolrServers(1, getClassName(), null); + } + + @AfterClass + private static void destroyData() + { + dismissSolrServers(); + } + + @After + public void clearData() throws Exception + { + deleteByQueryAllClients("*:*"); + } + + @Test + public void AlfrescoCollatableFieldType_emptyValuesSortingAsc__shouldBeRankedFirst() throws Exception { + prepareIndexSegmentWithAllNonNullFieldValues("text@s__sort@{http://www.alfresco.org/model/content/1.0}title"); + putHandleDefaults(); + // Docs with id 1, 3, 5 and 6 should be first (note that these will be sorted by indexing time). + String[] expectedRanking = new String[]{"1","3","5","6","4","2"}; + + QueryResponse response = query(getDefaultTestClient(), true, + "{\"query\":\"(id:(1 2 3 4 5 6))\",\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}], \"authorities\": [\"joel\"], \"tenants\": []}", + params("qt", "/afts", "shards.qt", "/afts", "start", "0", "rows", "100", "sort", "text@s__sort@{http://www.alfresco.org/model/content/1.0}title asc")); + + NamedList res = response.getResponse(); + SolrDocumentList searchResults = (SolrDocumentList)res.get("response"); + for(int i=0;i Date: Thu, 19 Dec 2019 22:13:05 +0000 Subject: [PATCH 069/129] Bump restapi from 1.25 to 1.26 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.25 to 1.26. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.25...v1.26) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 084518ad3..1eb5d0bf5 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.25 + 1.26 1.11 3.0.17 3.2.0 From fd95041243c0423882a0ebe17830497080aa489f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2019 22:13:30 +0000 Subject: [PATCH 070/129] Bump cmis from 1.11 to 1.12 in /e2e-test Bumps [cmis](https://github.com/Alfresco/alfresco-tas-cmis) from 1.11 to 1.12. - [Release notes](https://github.com/Alfresco/alfresco-tas-cmis/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-cmis/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-cmis/compare/v1.11...v1.12) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 084518ad3..01ea3495b 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -12,7 +12,7 @@ Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share 1.25 - 1.11 + 1.12 3.0.17 3.2.0 src/test/resources/SearchSuite.xml From 78fda408da7f96b76596800f99de0b4ec110c762 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2019 22:14:11 +0000 Subject: [PATCH 071/129] Bump alfresco-data-model from 8.70 to 8.72 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.70 to 8.72. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.70...8.72) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 82d0d83ff..a16c1574b 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.70 + 8.72 2.10.1 From f3b612334976877ce8db144b5a48af6eb39689d8 Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Fri, 20 Dec 2019 14:39:14 +0000 Subject: [PATCH 072/129] SEARCH-1894 Update Slave replica --- .../packaging/src/docker/search_config_setup.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/search_config_setup.sh b/search-services/packaging/src/docker/search_config_setup.sh index fb7636649..936a9768d 100644 --- a/search-services/packaging/src/docker/search_config_setup.sh +++ b/search-services/packaging/src/docker/search_config_setup.sh @@ -4,39 +4,53 @@ set -e # Slave replica service can be enabled using "REPLICATION_TYPE=slave" environment value. SOLR_CONFIG_FILE=$PWD/solrhome/templates/rerank/conf/solrconfig.xml + if [[ $REPLICATION_TYPE == "master" ]]; then - findStringMaster='/' + findStringMaster='' + replaceStringMaster="\n\t \n" + if [[ $REPLICATION_AFTER == "" ]]; then REPLICATION_AFTER=commit fi + for i in $(echo $REPLICATION_AFTER | sed "s/,/ /g") do replaceStringMaster+="\t\t"$i"<\/str> \n" done + if [[ ! -z "$REPLICATION_CONFIG_FILES" ]]; then replaceStringMaster+="\t\t$REPLICATION_CONFIG_FILES<\/str> \n" fi + replaceStringMaster+="\t<\/lst>" + sed -i "s/$findStringMaster/$findStringMaster$replaceStringMaster/g" $SOLR_CONFIG_FILE fi + if [[ $REPLICATION_TYPE == "slave" ]]; then + if [[ $REPLICATION_MASTER_PROTOCOL == "" ]]; then REPLICATION_MASTER_PROTOCOL=http fi + if [[ $REPLICATION_MASTER_HOST == "" ]]; then REPLICATION_MASTER_HOST=localhost fi + if [[ $REPLICATION_MASTER_PORT == "" ]]; then REPLICATION_MASTER_PORT=8083 fi + if [[ $REPLICATION_CORE_NAME == "" ]]; then REPLICATION_CORE_NAME=alfresco fi + if [[ $REPLICATION_POLL_INTERVAL == "" ]]; then REPLICATION_POLL_INTERVAL=00:00:30 fi + sed -i 's//\ \ '$REPLICATION_MASTER_PROTOCOL':\/\/'$REPLICATION_MASTER_HOST':'$REPLICATION_MASTER_PORT'\/solr\/'$REPLICATION_CORE_NAME'<\/str>\ From 7f59ae7d75677909286c29df53bd612f30256cbf Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Mon, 23 Dec 2019 12:18:50 +0000 Subject: [PATCH 073/129] SEARCH-1894 Update Slave replica --- .../packaging/src/docker/search_config_setup.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/search_config_setup.sh b/search-services/packaging/src/docker/search_config_setup.sh index 936a9768d..a4fcaf81c 100644 --- a/search-services/packaging/src/docker/search_config_setup.sh +++ b/search-services/packaging/src/docker/search_config_setup.sh @@ -4,6 +4,7 @@ set -e # Slave replica service can be enabled using "REPLICATION_TYPE=slave" environment value. SOLR_CONFIG_FILE=$PWD/solrhome/templates/rerank/conf/solrconfig.xml +SOLR_CORE_FILE=$PWD/solrhome/templates/rerank/conf/solrcore.properties if [[ $REPLICATION_TYPE == "master" ]]; then @@ -12,7 +13,11 @@ if [[ $REPLICATION_TYPE == "master" ]]; then replaceStringMaster="\n\t \n" if [[ $REPLICATION_AFTER == "" ]]; then - REPLICATION_AFTER=commit + REPLICATION_AFTER=commit,startup + fi + + if [[ $REPLICATION_CONFIG_FILES == "" ]]; then + REPLICATION_CONFIG_FILES=schema.xml,stopwords.txt fi for i in $(echo $REPLICATION_AFTER | sed "s/,/ /g") @@ -27,6 +32,7 @@ if [[ $REPLICATION_TYPE == "master" ]]; then replaceStringMaster+="\t<\/lst>" sed -i "s/$findStringMaster/$findStringMaster$replaceStringMaster/g" $SOLR_CONFIG_FILE + sed -i "s/enable.alfresco.tracking=true/enable.alfresco.tracking=true\nenable.master=true\nenable.slave=false/g" $SOLR_CORE_FILE fi if [[ $REPLICATION_TYPE == "slave" ]]; then @@ -56,6 +62,7 @@ if [[ $REPLICATION_TYPE == "slave" ]]; then '$REPLICATION_MASTER_PROTOCOL':\/\/'$REPLICATION_MASTER_HOST':'$REPLICATION_MASTER_PORT'\/solr\/'$REPLICATION_CORE_NAME'<\/str>\ '$REPLICATION_POLL_INTERVAL'<\/str>\ <\/lst>/g' $SOLR_CONFIG_FILE + sed -i "s/enable.alfresco.tracking=true/enable.alfresco.tracking=false\nenable.master=false\nenable.slave=true/g" $SOLR_CORE_FILE fi SOLR_IN_FILE=$PWD/solr.in.sh From d6faa32dec7f7df675cab0e4ac8a8a5f1b55e3b8 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Fri, 27 Dec 2019 13:26:55 +0100 Subject: [PATCH 074/129] Delete nodes only if they exist --- .../alfresco/solr/SolrInformationServer.java | 43 +++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index e545e35b4..8a784b734 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -164,6 +164,8 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DelegatingCollector; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.QueryCommand; +import org.apache.solr.search.QueryResult; import org.apache.solr.search.QueryWrapperFilter; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.update.AddUpdateCommand; @@ -2190,13 +2192,26 @@ public class SolrInformationServer implements InformationServer } } } - + private void deleteErrorNode(UpdateRequestProcessor processor, SolrQueryRequest request, Node node) throws IOException { + String errorDocId = PREFIX_ERROR + node.getId(); - DeleteUpdateCommand delErrorDocCmd = new DeleteUpdateCommand(request); - delErrorDocCmd.setId(errorDocId); - processor.processDelete(delErrorDocCmd); + + // Try finding the node before performing removal operation + QueryResult result = new QueryResult(); + Query query = new TermQuery(new Term("id", errorDocId)); + QueryCommand queryCommand = new QueryCommand(); + queryCommand.setQuery(query); + core.getSearcher().get().search(result, queryCommand); + + if (result.getDocList().size() > 0) + { + DeleteUpdateCommand delErrorDocCmd = new DeleteUpdateCommand(request); + delErrorDocCmd.setId(errorDocId); + processor.processDelete(delErrorDocCmd); + } + } @@ -2208,12 +2223,24 @@ public class SolrInformationServer implements InformationServer // MNT-13767 fix, remove by node DBID. deleteNode(processor, request, node.getId()); } - + private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, long dbid) throws IOException { - DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request); - delDocCmd.setQuery(FIELD_DBID + ":" + dbid); - processor.processDelete(delDocCmd); + + // Try finding the node before performing removal operation + QueryResult result = new QueryResult(); + Query query = new TermQuery(new Term(FIELD_DBID, String.valueOf(dbid))); + QueryCommand queryCommand = new QueryCommand(); + queryCommand.setQuery(query); + core.getSearcher().get().search(result, queryCommand); + + if (result.getDocList().size() > 0) + { + DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request); + delDocCmd.setQuery(FIELD_DBID + ":" + dbid); + processor.processDelete(delDocCmd); + } + } private boolean isContentIndexedForNode(Map properties) From 15f2c2274a2cdc137ed14f761cf18e39d63edbf9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2019 22:13:00 +0000 Subject: [PATCH 075/129] Bump alfresco-data-model from 8.72 to 8.73 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.72 to 8.73. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.72...8.73) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index a16c1574b..0e2038bde 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.72 + 8.73 2.10.1 From 1237f27ad9330fd5f6a03c484b316533c363f5b6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2019 22:16:24 +0000 Subject: [PATCH 076/129] Bump randomizedtesting-runner from 2.7.5 to 2.7.6 in /search-services Bumps randomizedtesting-runner from 2.7.5 to 2.7.6. Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 34d4aed4c..50c987a82 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -171,7 +171,7 @@ com.carrotsearch.randomizedtesting randomizedtesting-runner - 2.7.5 + 2.7.6 test From 7dd4c085f95113773f14b4db203c136f7777640d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2020 22:13:39 +0000 Subject: [PATCH 077/129] Bump junit from 4.12 to 4.13 in /search-services Bumps [junit](https://github.com/junit-team/junit4) from 4.12 to 4.13. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/master/doc/ReleaseNotes4.12.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.12...r4.13) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- search-services/alfresco-solrclient-lib/pom.xml | 2 +- search-services/packaging/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 34d4aed4c..6cf42a434 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -151,7 +151,7 @@ junit junit - 4.12 + 4.13 test diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index a16c1574b..7d08510ed 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -61,7 +61,7 @@ junit junit - 4.12 + 4.13 test diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index bf970d553..1ed5774f9 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -33,7 +33,7 @@ junit junit - 4.12 + 4.13 test From 84793aedba5ba26e70ee101eaf60e8c5eb00a714 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 3 Jan 2020 22:12:58 +0000 Subject: [PATCH 078/129] Bump alfresco-data-model from 8.72 to 8.74 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.72 to 8.74. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.72...8.74) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index a16c1574b..b9b325201 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.72 + 8.74 2.10.1 From fa076cf52dfc5ab6341ffacaf5d90b9ba304882f Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 6 Jan 2020 13:57:08 +0000 Subject: [PATCH 079/129] Add CONTRIBUTING.md. --- CONTRIBUTING.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 6 +++++- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..cf3bb9b4f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing + +Thanks for your interest in contributing to this project! + +The following is a set of guidelines for contributing to this library. Most of them will +make the life of the reviewer easier and therefore decrease the time required for the +patch be included in the next version. + +Alfresco has an [active forum](http://community.alfresco.com/community/ecm) to support +community users of our products. If you have any questions then this is the fastest method +of getting an answer. + +We have a [coding standards guidelines page](https://hub.alfresco.com/t5/alfresco-content-services-hub/coding-standards-for-alfresco-content-services/ba-p/290457) +although you will find numerous examples where we have not adhered to them. Please try to +maintain consistency with the guidelines for new code, but avoid reformatting large +blocks of code if these are not related to your change. + +## Branches + +Our codebase consists of long-lived release branches and short-lived feature branches. The +code that we expect to include in the next minor version is stored on `master`. All other +release branches have the prefix `release/`. Feature branches may have any other prefix, +but we usually use `feature/` or `fix/`. We expect code on release branches to be ready +to release, and in the rare occasion when a release branch is broken then we try to revert +changes to fix the branch as soon as possible. + +As bug fixes often also need a change to ACS then we use a cherry-pick strategy to get the +fix to all necessary release branches. The fix should initially be merged to `master` and +it can then be cherry-picked back by using: + +```git cherry-pick -x -m 1 [mergeCommitId]``` + +## Community Mirror + +Pull requests to our community mirror will be accepted in our enterprise codebase and then +mirrored back to the community. You will always be credited with your commits, although if +you [sign your commits](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) then the +signature will be stripped by the mirroring process.[^dependabot] + +[^dependabot]: This is the reason that pull requests submitted by Dependabot appear closed +rather than merged. + +## Builds + +Our builds are currently in our internal Bamboo instance. We have an [internal dashboard](http://pson01.alfresco.com:8081/SAI-FeatureBranches) +displaying the status of all branches, and also an [internal compatibility dashboard](http://pson01.alfresco.com:8081/SAI-Compatibility) +displaying results of integration testing with different versions of ACS. + +Our build process uses the scripts in the [build scripts](https://git.alfresco.com/search_discovery/BuildScripts) +project. + +Although the build results are not visible externally, it should be possible to run most of +the tests locally. We have divided our tests into unit tests, integration tests and +end-to-end tests. The unit and integration tests can be run using the maven `test` and +`verify` goals respectively. The end-to-end tests cannot currently be run externally as +they require some dependencies stored in our internal Nexus. diff --git a/README.md b/README.md index 95ab30356..b973c9523 100644 --- a/README.md +++ b/README.md @@ -40,4 +40,8 @@ More details are available at [search-services](/search-services) folder. **Following resources will not be available for Community users** -More details are available at [insight-engine](/insight-engine) folder. \ No newline at end of file +More details are available at [insight-engine](/insight-engine) folder. + +## Contributing guide + +Please use [this guide](CONTRIBUTING.md) to make a contribution to the project. From df218492b086034d74f19b00983f31deaa33dd6f Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 6 Jan 2020 14:44:32 +0000 Subject: [PATCH 080/129] Move some guidelines to internal file. Also add details about raising issues. --- CONTRIBUTING.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf3bb9b4f..575155bab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,21 @@ Alfresco has an [active forum](http://community.alfresco.com/community/ecm) to s community users of our products. If you have any questions then this is the fastest method of getting an answer. +## Raising issues + +If you want to raise an issue then please use the [issue tracker on GitHub](https://github.com/Alfresco/SearchServices/issues). +We may convert these to Jira tickets before working on them, as this is the system we use +internally for tracking development. The Jira project for this codebase is [SEARCH](https://issues.alfresco.com/jira/projects/SEARCH/issues) +and you may want to look here first to see if your issue has previously been encountered.[^jiraaccess] +There are some guidelines for raising a good issue [here](https://hub.alfresco.com/t5/alfresco-content-services-hub/reporting-an-issue/ba-p/289727). + +[^jiraaccess]: Note that while we try to keep our Jira issues visible to everyone, some +are restricted as they relate to specific customers or security issues. Some older issues +are also restricted simply because we have not been through to check if they contain +sensitive information or not. + +## Submitting changes + We have a [coding standards guidelines page](https://hub.alfresco.com/t5/alfresco-content-services-hub/coding-standards-for-alfresco-content-services/ba-p/290457) although you will find numerous examples where we have not adhered to them. Please try to maintain consistency with the guidelines for new code, but avoid reformatting large @@ -42,12 +57,8 @@ rather than merged. ## Builds -Our builds are currently in our internal Bamboo instance. We have an [internal dashboard](http://pson01.alfresco.com:8081/SAI-FeatureBranches) -displaying the status of all branches, and also an [internal compatibility dashboard](http://pson01.alfresco.com:8081/SAI-Compatibility) -displaying results of integration testing with different versions of ACS. - -Our build process uses the scripts in the [build scripts](https://git.alfresco.com/search_discovery/BuildScripts) -project. +Our builds are currently in our internal Bamboo instance. If you have access to the +internal code then you can find some more links in the [insight engine module](insight-engine/CONTRIBUTING.md). Although the build results are not visible externally, it should be possible to run most of the tests locally. We have divided our tests into unit tests, integration tests and From 7159a6087881d89ddd295cb53e7dac9c281e8864 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2020 14:45:47 +0000 Subject: [PATCH 081/129] Bump alfresco-data-model from 8.73 to 8.75 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.73 to 8.75. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.73...8.75) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 4f7e6bc7e..edcea929b 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.74 + 8.75 2.10.1 From 85632973a5d88f55b473b98c89b297bfdb04eacc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2020 22:12:21 +0000 Subject: [PATCH 082/129] Bump dependency.jackson.version in /search-services Bumps `dependency.jackson.version` from 2.10.1 to 2.10.2. Updates `jackson-core` from 2.10.1 to 2.10.2 - [Release notes](https://github.com/FasterXML/jackson-core/releases) - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.10.1...jackson-core-2.10.2) Updates `jackson-annotations` from 2.10.1 to 2.10.2 - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) Updates `jackson-databind` from 2.10.1 to 2.10.2 - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 4f7e6bc7e..e728a93d4 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -23,7 +23,7 @@ 8.74 - 2.10.1 + 2.10.2 From d0fe6a380f74cce9d50126a32d0fc06a8520f996 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 7 Jan 2020 08:04:33 +0000 Subject: [PATCH 083/129] Update notice.txt for jackson 2.10.2. --- .../packaging/src/main/resources/licenses/notice.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 9932d0653..3ff7a566b 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -54,9 +54,9 @@ mybatis-spring-1.2.5.jar http://www.mybatis.org/ chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/ chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/ quartz-2.3.2.jar http://quartz-scheduler.org/ -jackson-core-2.10.1.jar https://github.com/FasterXML/jackson -jackson-annotations-2.10.1.jar https://github.com/FasterXML/jackson -jackson-databind-2.10.1.jar https://github.com/FasterXML/jackson +jackson-core-2.10.2.jar https://github.com/FasterXML/jackson +jackson-annotations-2.10.2.jar https://github.com/FasterXML/jackson +jackson-databind-2.10.2.jar https://github.com/FasterXML/jackson commons-httpclient-3.1-HTTPCLIENT-1265.jar http://jakarta.apache.org/commons/ spring-aop-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ spring-beans-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ From f29e3250fd2ba4dfbb8c415bd272d53c7a55ef18 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 7 Jan 2020 11:18:18 +0100 Subject: [PATCH 084/129] Skip long periods of time where repository is not ingesting new content. --- .../solr/tracker/MetadataTracker.java | 11 +++++ .../alfresco/solr/client/SOLRAPIClient.java | 49 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 3862ebd30..7c0df7841 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -546,6 +546,17 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate); startTime += actualTimeStep; + + // If no transactions are found, advance the time window to the next available transaction commit time + if (transactions.getTransactions().size() == 0) + { + Long nextTxCommitTime = client.getNextTxCommitTime(coreName, startTime); + if (nextTxCommitTime != -1) + { + log.info("Advancing transactions from startTime = " + startTime + " to " + nextTxCommitTime); + transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate); + } + } } while (((transactions.getTransactions().size() == 0) && (startTime < endTime)) || ((transactions.getTransactions().size() > 0) && alreadyFoundTransactions(txnsFound, transactions))); diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 136d5ddde..2be8197e1 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -103,7 +103,8 @@ public class SOLRAPIClient private static final String GET_NODES_URL = "api/solr/nodes"; private static final String GET_CONTENT = "api/solr/textContent"; private static final String GET_MODEL = "api/solr/model"; - private static final String GET_MODELS_DIFF = "api/solr/modelsdiff"; + private static final String GET_MODELS_DIFF = "api/solr/modelsdiff"; + private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction"; private static final String CHECKSUM_HEADER = "XAlfresco-modelChecksum"; @@ -1228,7 +1229,51 @@ public class SOLRAPIClient } return diffs; - } + } + + /** + * Returns the minimum and the maximum commit time for transactions in a node id range. + * + * @param coreName alfresco, archive + * @param fromCommitTime initial transaction commit time + * @return Time of the next transaction + * @throws IOException + * @throws AuthenticationException + * @throws Exception + */ + public Long getNextTxCommitTime(String coreName, Long fromCommitTime) throws AuthenticationException, IOException + { + StringBuilder url = new StringBuilder(GET_NEXT_TX_COMMIT_TIME); + url.append("?").append("fromCommitTime").append("=").append(fromCommitTime); + GetRequest get = new GetRequest(url.toString()); + Response response = null; + JSONObject json = null; + try + { + response = repositoryHttpClient.sendRequest(get); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new AlfrescoRuntimeException(coreName + " - GetNextTxCommitTime return status is " + + response.getStatus() + " when invoking " + url); + } + + Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); + json = new JSONObject(new JSONTokener(reader)); + } + finally + { + if (response != null) + { + response.release(); + } + } + if (log.isDebugEnabled()) + { + log.debug(json.toString()); + } + + return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString()); + } /* * type conversions from serialized JSON values to SOLR-consumable objects From 0afe8e7b85d9d4fd2a83ca408345799e7d8477c0 Mon Sep 17 00:00:00 2001 From: sridharvellingiri Date: Tue, 7 Jan 2020 14:07:29 +0000 Subject: [PATCH 085/129] SEARCH-1894 Update search config script with noRerank template --- .../templates/noRerank/conf/solrconfig.xml | 2 +- .../src/docker/search_config_setup.sh | 19 ++++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml index a51492e56..e7bd84f6c 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml @@ -1154,7 +1154,7 @@ https://wiki.apache.org/solr/SolrCloud/ --> - + + 1.5.0 unpack-solr-war From d28f05b19be51ef8d7a13e928db024814d4b3fe9 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 14 Jan 2020 16:50:16 +0100 Subject: [PATCH 095/129] Use gzipped compressed streams to recover content text from repository. This modification will also require Tomcat / NGINX configuration in order to accept HTTP GZIP requests. --- .../alfresco/solr/SolrInformationServer.java | 7 +++++- .../alfresco/solr/client/SOLRAPIClient.java | 24 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index c5b07d69f..7c8552cb5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -81,6 +81,7 @@ import java.util.Map.Entry; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; import com.carrotsearch.hppc.IntArrayList; @@ -2498,8 +2499,12 @@ public class SolrInformationServer implements InformationServer response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME, response); - + InputStream ris = response.getContent(); + if (null != response.getContentEncoding() && response.getContentEncoding().equals("gzip")) + { + ris = new GZIPInputStream(ris); + } String textContent = ""; try { diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 136d5ddde..1bf4e9954 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -73,6 +74,7 @@ import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.util.DateUtil; +import org.apache.commons.io.IOUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -1118,20 +1120,21 @@ public class SOLRAPIClient GetRequest req = new GetRequest(url.toString()); + Map headers = new HashMap(); if(modifiedSince != null) { - Map headers = new HashMap(1, 1.0f); headers.put("If-Modified-Since", String.valueOf(DateUtil.formatDate(new Date(modifiedSince)))); - req.setHeaders(headers); } - + headers.put("Accept-Encoding", "gzip"); + req.setHeaders(headers); + Response response = repositoryHttpClient.sendRequest(req); if(response.getStatus() != Status.STATUS_NOT_MODIFIED && response.getStatus() != Status.STATUS_NO_CONTENT && response.getStatus() != Status.STATUS_OK) { throw new AlfrescoRuntimeException("GetTextContentResponse return status is " + response.getStatus()); - } - + } + return new GetTextContentResponse(response); } @@ -1481,7 +1484,8 @@ public class SOLRAPIClient private SolrApiContentStatus status; private String transformException; private String transformStatusStr; - private Long transformDuration; + private Long transformDuration; + private String contentEncoding; public GetTextContentResponse(Response response) throws IOException { @@ -1491,7 +1495,8 @@ public class SOLRAPIClient this.transformStatusStr = response.getHeader("X-Alfresco-transformStatus"); this.transformException = response.getHeader("X-Alfresco-transformException"); String tmp = response.getHeader("X-Alfresco-transformDuration"); - this.transformDuration = (tmp != null ? Long.valueOf(tmp) : null); + this.transformDuration = (tmp != null ? Long.valueOf(tmp) : null); + this.contentEncoding = response.getHeader("Content-Encoding"); setStatus(); } @@ -1557,6 +1562,11 @@ public class SOLRAPIClient public Long getTransformDuration() { return transformDuration; + } + + public String getContentEncoding() + { + return contentEncoding; } } From 8747a4827b5ef7c8b59d84a6a39856e5e7b666f2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2020 22:13:50 +0000 Subject: [PATCH 096/129] Bump utility from 3.0.17 to 3.0.18 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.17 to 3.0.18. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.17...utility-3.0.18) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 3bd320096..df6414653 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 1.26 1.12 - 3.0.17 + 3.0.18 3.3.0 src/test/resources/SearchSuite.xml From 9ff35e7807dff85de07e6f99d79b1c2d6bbe77c9 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Thu, 16 Jan 2020 11:00:25 +0100 Subject: [PATCH 097/129] Add this feature to SOLR Core properties, in order to allow the user to switch on the compression depending on his environment and use case. --- .../noRerank/conf/solrcore.properties | 8 ++++++ .../templates/rerank/conf/solrcore.properties | 7 +++++ .../alfresco/solr/client/SOLRAPIClient.java | 28 ++++++++++++++++--- .../solr/client/SOLRAPIClientFactory.java | 3 +- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index e91ba7ee8..a6a5a1fae 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -174,6 +174,14 @@ solr.suggester.enabled=true # -1 to disable suggester build throttling solr.suggester.minSecsBetweenBuilds=3600 +# +# Request content text compression +# When enabling this option, Tomcat Connector or HTTP Web Proxy (NGINX, Apache) compression must be also enabled +# This setting can improve performance when having high network latency or large documents in the repository +# +solr.request.content.compress=false + + # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 06f089370..fdf702da6 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -174,6 +174,13 @@ solr.suggester.enabled=true # -1 to disable suggester build throttling solr.suggester.minSecsBetweenBuilds=3600 +# +# Request content text compression +# When enabling this option, Tomcat Connector or HTTP Web Proxy (NGINX, Apache) compression must be also enabled +# This setting can improve performance when having high network latency or large documents in the repository +# +solr.request.content.compress=false + # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 1bf4e9954..b3278a273 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -113,17 +113,34 @@ public class SOLRAPIClient private SOLRDeserializer deserializer; private DictionaryService dictionaryService; private JsonFactory jsonFactory; - private NamespaceDAO namespaceDAO; + private NamespaceDAO namespaceDAO; + + /** + * This option enables ("Accept-Encoding": "gzip") header for compression + * in GET_CONTENT requests. Additional configuration is required in + * Alfresco Repository Tomcat Connector or HTTP Web Proxy to deal w + * with compressed requests. + */ + private boolean compression; + public SOLRAPIClient(AlfrescoHttpClient repositoryHttpClient, + DictionaryService dictionaryService, + NamespaceDAO namespaceDAO) + { + this(repositoryHttpClient, dictionaryService, namespaceDAO, false); + } + public SOLRAPIClient(AlfrescoHttpClient repositoryHttpClient, DictionaryService dictionaryService, - NamespaceDAO namespaceDAO) + NamespaceDAO namespaceDAO, + boolean compression) { this.repositoryHttpClient = repositoryHttpClient; this.dictionaryService = dictionaryService; this.namespaceDAO = namespaceDAO; this.deserializer = new SOLRDeserializer(namespaceDAO); - this.jsonFactory = new JsonFactory(); + this.jsonFactory = new JsonFactory(); + this.compression = compression; } /** @@ -1125,7 +1142,10 @@ public class SOLRAPIClient { headers.put("If-Modified-Since", String.valueOf(DateUtil.formatDate(new Date(modifiedSince)))); } - headers.put("Accept-Encoding", "gzip"); + if (compression) + { + headers.put("Accept-Encoding", "gzip"); + } req.setHeaders(headers); Response response = repositoryHttpClient.sendRequest(req); diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java index 0fbf5df7a..f467d3242 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java @@ -143,6 +143,7 @@ public class SOLRAPIClientFactory alfrescoHost = props.getProperty("alfresco.host", "localhost"); alfrescoPort = Integer.parseInt(props.getProperty("alfresco.port", "8080")); alfrescoPortSSL = Integer.parseInt(props.getProperty("alfresco.port.ssl", "8443")); + boolean compression = Boolean.parseBoolean(props.getProperty("solr.request.content.compress", "false")); SOLRAPIClient client = getCachedClient(alfrescoHost, alfrescoPort, alfrescoPortSSL); if (client == null) @@ -171,7 +172,7 @@ public class SOLRAPIClientFactory maxHostConnections = Integer.parseInt(props.getProperty("alfresco.maxHostConnections", "40")); socketTimeout = Integer.parseInt(props.getProperty("alfresco.socketTimeout", "60000")); - client = new SOLRAPIClient(getRepoClient(keyResourceLoader), dictionaryService, namespaceDAO); + client = new SOLRAPIClient(getRepoClient(keyResourceLoader), dictionaryService, namespaceDAO, compression); setCachedClient(alfrescoHost, alfrescoPort, alfrescoPortSSL, client); } From de1b47a2e0885afac89c198719b81c59242e3d2b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2020 22:13:51 +0000 Subject: [PATCH 098/129] Bump cmis from 1.12 to 1.13 in /e2e-test Bumps [cmis](https://github.com/Alfresco/alfresco-tas-cmis) from 1.12 to 1.13. - [Release notes](https://github.com/Alfresco/alfresco-tas-cmis/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-cmis/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-cmis/compare/v1.12...v1.13) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 3bd320096..4fcb122b4 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -12,7 +12,7 @@ Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share 1.26 - 1.12 + 1.13 3.0.17 3.3.0 src/test/resources/SearchSuite.xml From 73e465567c5044d1d0f1c89b44a6f67df48caff2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2020 22:14:25 +0000 Subject: [PATCH 099/129] Bump restapi from 1.26 to 1.28 in /e2e-test Bumps [restapi](https://github.com/Alfresco/alfresco-tas-restapi) from 1.26 to 1.28. - [Release notes](https://github.com/Alfresco/alfresco-tas-restapi/releases) - [Commits](https://github.com/Alfresco/alfresco-tas-restapi/compare/v1.26...v1.28) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 3bd320096..dca709884 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.26 + 1.28 1.12 3.0.17 3.3.0 From 2c5ab74dc59750ffa3be5e20e9f36840d5176781 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 17 Jan 2020 08:51:38 +0000 Subject: [PATCH 100/129] Update URL for maven-restlet repository. Restlet were acquired by Talend in 2017 and the old URL no longer works. --- search-services/alfresco-search/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 2916a884e..752f9768b 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -182,6 +182,14 @@ + + + maven-restlet + Public online Restlet repository + http://maven.restlet.talend.com + + + alfresco-solr From 66e7cb9ff2a8131ee2b44ab19e0c52afac17cb66 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 17 Jan 2020 09:27:19 +0000 Subject: [PATCH 101/129] SEARCH-2054 Decrease the priority of the restlet repository. This ensures that artifacts are only downloaded from maven-restlet if they aren't found in an alfresco or central repo. --- search-services/alfresco-search/pom.xml | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 752f9768b..1bffd3a2b 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -183,6 +183,35 @@ + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + true + + + false + + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/groups/public-snapshots + + false + + + true + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + maven-restlet Public online Restlet repository From ee21af7b2706b084e3b87bf56e9443325b52b57c Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Fri, 17 Jan 2020 14:25:59 +0100 Subject: [PATCH 102/129] Fix changes from review. --- .../main/java/org/alfresco/solr/SolrInformationServer.java | 2 +- .../src/main/java/org/alfresco/solr/client/SOLRAPIClient.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 7c8552cb5..a4ea83035 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -2501,7 +2501,7 @@ public class SolrInformationServer implements InformationServer response); InputStream ris = response.getContent(); - if (null != response.getContentEncoding() && response.getContentEncoding().equals("gzip")) + if (response.getContentEncoding().equals("gzip")) { ris = new GZIPInputStream(ris); } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index b3278a273..ebc44dbd0 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -118,7 +118,7 @@ public class SOLRAPIClient /** * This option enables ("Accept-Encoding": "gzip") header for compression * in GET_CONTENT requests. Additional configuration is required in - * Alfresco Repository Tomcat Connector or HTTP Web Proxy to deal w + * Alfresco Repository Tomcat Connector or HTTP Web Proxy to deal * with compressed requests. */ private boolean compression; @@ -1137,7 +1137,7 @@ public class SOLRAPIClient GetRequest req = new GetRequest(url.toString()); - Map headers = new HashMap(); + Map headers = new HashMap<>(); if(modifiedSince != null) { headers.put("If-Modified-Since", String.valueOf(DateUtil.formatDate(new Date(modifiedSince)))); From 8dd9daa1996f90042e56ec0019ca1770707f608a Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Fri, 17 Jan 2020 15:24:57 +0000 Subject: [PATCH 103/129] Revert "Merge branch 'feature/SEARCH-2029_ContentResponseCompression' into 'master'" This reverts merge request !320 --- .../alfresco/solr/SolrInformationServer.java | 7 +-- .../noRerank/conf/solrcore.properties | 8 --- .../templates/rerank/conf/solrcore.properties | 7 --- .../alfresco/solr/client/SOLRAPIClient.java | 50 ++++--------------- .../solr/client/SOLRAPIClientFactory.java | 3 +- 5 files changed, 12 insertions(+), 63 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index a4ea83035..c5b07d69f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -81,7 +81,6 @@ import java.util.Map.Entry; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; import com.carrotsearch.hppc.IntArrayList; @@ -2499,12 +2498,8 @@ public class SolrInformationServer implements InformationServer response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME, response); - + InputStream ris = response.getContent(); - if (response.getContentEncoding().equals("gzip")) - { - ris = new GZIPInputStream(ris); - } String textContent = ""; try { diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index a6a5a1fae..e91ba7ee8 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -174,14 +174,6 @@ solr.suggester.enabled=true # -1 to disable suggester build throttling solr.suggester.minSecsBetweenBuilds=3600 -# -# Request content text compression -# When enabling this option, Tomcat Connector or HTTP Web Proxy (NGINX, Apache) compression must be also enabled -# This setting can improve performance when having high network latency or large documents in the repository -# -solr.request.content.compress=false - - # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index fdf702da6..06f089370 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -174,13 +174,6 @@ solr.suggester.enabled=true # -1 to disable suggester build throttling solr.suggester.minSecsBetweenBuilds=3600 -# -# Request content text compression -# When enabling this option, Tomcat Connector or HTTP Web Proxy (NGINX, Apache) compression must be also enabled -# This setting can improve performance when having high network latency or large documents in the repository -# -solr.request.content.compress=false - # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index ebc44dbd0..136d5ddde 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -74,7 +73,6 @@ import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.util.DateUtil; -import org.apache.commons.io.IOUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -113,34 +111,17 @@ public class SOLRAPIClient private SOLRDeserializer deserializer; private DictionaryService dictionaryService; private JsonFactory jsonFactory; - private NamespaceDAO namespaceDAO; - - /** - * This option enables ("Accept-Encoding": "gzip") header for compression - * in GET_CONTENT requests. Additional configuration is required in - * Alfresco Repository Tomcat Connector or HTTP Web Proxy to deal - * with compressed requests. - */ - private boolean compression; + private NamespaceDAO namespaceDAO; - public SOLRAPIClient(AlfrescoHttpClient repositoryHttpClient, - DictionaryService dictionaryService, - NamespaceDAO namespaceDAO) - { - this(repositoryHttpClient, dictionaryService, namespaceDAO, false); - } - public SOLRAPIClient(AlfrescoHttpClient repositoryHttpClient, DictionaryService dictionaryService, - NamespaceDAO namespaceDAO, - boolean compression) + NamespaceDAO namespaceDAO) { this.repositoryHttpClient = repositoryHttpClient; this.dictionaryService = dictionaryService; this.namespaceDAO = namespaceDAO; this.deserializer = new SOLRDeserializer(namespaceDAO); - this.jsonFactory = new JsonFactory(); - this.compression = compression; + this.jsonFactory = new JsonFactory(); } /** @@ -1137,24 +1118,20 @@ public class SOLRAPIClient GetRequest req = new GetRequest(url.toString()); - Map headers = new HashMap<>(); if(modifiedSince != null) { + Map headers = new HashMap(1, 1.0f); headers.put("If-Modified-Since", String.valueOf(DateUtil.formatDate(new Date(modifiedSince)))); + req.setHeaders(headers); } - if (compression) - { - headers.put("Accept-Encoding", "gzip"); - } - req.setHeaders(headers); - + Response response = repositoryHttpClient.sendRequest(req); if(response.getStatus() != Status.STATUS_NOT_MODIFIED && response.getStatus() != Status.STATUS_NO_CONTENT && response.getStatus() != Status.STATUS_OK) { throw new AlfrescoRuntimeException("GetTextContentResponse return status is " + response.getStatus()); - } - + } + return new GetTextContentResponse(response); } @@ -1504,8 +1481,7 @@ public class SOLRAPIClient private SolrApiContentStatus status; private String transformException; private String transformStatusStr; - private Long transformDuration; - private String contentEncoding; + private Long transformDuration; public GetTextContentResponse(Response response) throws IOException { @@ -1515,8 +1491,7 @@ public class SOLRAPIClient this.transformStatusStr = response.getHeader("X-Alfresco-transformStatus"); this.transformException = response.getHeader("X-Alfresco-transformException"); String tmp = response.getHeader("X-Alfresco-transformDuration"); - this.transformDuration = (tmp != null ? Long.valueOf(tmp) : null); - this.contentEncoding = response.getHeader("Content-Encoding"); + this.transformDuration = (tmp != null ? Long.valueOf(tmp) : null); setStatus(); } @@ -1582,11 +1557,6 @@ public class SOLRAPIClient public Long getTransformDuration() { return transformDuration; - } - - public String getContentEncoding() - { - return contentEncoding; } } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java index f467d3242..0fbf5df7a 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java @@ -143,7 +143,6 @@ public class SOLRAPIClientFactory alfrescoHost = props.getProperty("alfresco.host", "localhost"); alfrescoPort = Integer.parseInt(props.getProperty("alfresco.port", "8080")); alfrescoPortSSL = Integer.parseInt(props.getProperty("alfresco.port.ssl", "8443")); - boolean compression = Boolean.parseBoolean(props.getProperty("solr.request.content.compress", "false")); SOLRAPIClient client = getCachedClient(alfrescoHost, alfrescoPort, alfrescoPortSSL); if (client == null) @@ -172,7 +171,7 @@ public class SOLRAPIClientFactory maxHostConnections = Integer.parseInt(props.getProperty("alfresco.maxHostConnections", "40")); socketTimeout = Integer.parseInt(props.getProperty("alfresco.socketTimeout", "60000")); - client = new SOLRAPIClient(getRepoClient(keyResourceLoader), dictionaryService, namespaceDAO, compression); + client = new SOLRAPIClient(getRepoClient(keyResourceLoader), dictionaryService, namespaceDAO); setCachedClient(alfrescoHost, alfrescoPort, alfrescoPortSSL, client); } From 641bb26dcc0a16f56689b239c8bcc6fe4c6ba202 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Mon, 20 Jan 2020 10:18:19 +0100 Subject: [PATCH 104/129] Fix null pointer comparison --- .../src/main/java/org/alfresco/solr/SolrInformationServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index a4ea83035..182067271 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -2501,7 +2501,7 @@ public class SolrInformationServer implements InformationServer response); InputStream ris = response.getContent(); - if (response.getContentEncoding().equals("gzip")) + if (Objects.equals(response.getContentEncoding(), "gzip")) { ris = new GZIPInputStream(ris); } From c1e660319b6eff8c230bb9979c90fda4b1e4ca17 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 20 Jan 2020 09:58:42 +0000 Subject: [PATCH 105/129] Revert "Revert "Merge branch 'feature/SEARCH-2029_ContentResponseCompression' into 'master'"" This reverts commit 221567ba8c0ef24eb70062f8182c14227fa6a789. --- .../alfresco/solr/SolrInformationServer.java | 3 +- .../noRerank/conf/solrcore.properties | 8 +++ .../templates/rerank/conf/solrcore.properties | 7 +++ .../alfresco/solr/client/SOLRAPIClient.java | 50 +++++++++++++++---- .../solr/client/SOLRAPIClientFactory.java | 3 +- 5 files changed, 59 insertions(+), 12 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 1f5a616c7..182067271 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -81,6 +81,7 @@ import java.util.Map.Entry; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; import com.carrotsearch.hppc.IntArrayList; @@ -2498,7 +2499,7 @@ public class SolrInformationServer implements InformationServer response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME, response); - + InputStream ris = response.getContent(); if (Objects.equals(response.getContentEncoding(), "gzip")) { diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties index e91ba7ee8..a6a5a1fae 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties @@ -174,6 +174,14 @@ solr.suggester.enabled=true # -1 to disable suggester build throttling solr.suggester.minSecsBetweenBuilds=3600 +# +# Request content text compression +# When enabling this option, Tomcat Connector or HTTP Web Proxy (NGINX, Apache) compression must be also enabled +# This setting can improve performance when having high network latency or large documents in the repository +# +solr.request.content.compress=false + + # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 06f089370..fdf702da6 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -174,6 +174,13 @@ solr.suggester.enabled=true # -1 to disable suggester build throttling solr.suggester.minSecsBetweenBuilds=3600 +# +# Request content text compression +# When enabling this option, Tomcat Connector or HTTP Web Proxy (NGINX, Apache) compression must be also enabled +# This setting can improve performance when having high network latency or large documents in the repository +# +solr.request.content.compress=false + # # Limit the maximum text size of transformed content sent to the index - in bytes # diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 136d5ddde..ebc44dbd0 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -73,6 +74,7 @@ import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.util.DateUtil; +import org.apache.commons.io.IOUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -111,17 +113,34 @@ public class SOLRAPIClient private SOLRDeserializer deserializer; private DictionaryService dictionaryService; private JsonFactory jsonFactory; - private NamespaceDAO namespaceDAO; + private NamespaceDAO namespaceDAO; + + /** + * This option enables ("Accept-Encoding": "gzip") header for compression + * in GET_CONTENT requests. Additional configuration is required in + * Alfresco Repository Tomcat Connector or HTTP Web Proxy to deal + * with compressed requests. + */ + private boolean compression; + public SOLRAPIClient(AlfrescoHttpClient repositoryHttpClient, + DictionaryService dictionaryService, + NamespaceDAO namespaceDAO) + { + this(repositoryHttpClient, dictionaryService, namespaceDAO, false); + } + public SOLRAPIClient(AlfrescoHttpClient repositoryHttpClient, DictionaryService dictionaryService, - NamespaceDAO namespaceDAO) + NamespaceDAO namespaceDAO, + boolean compression) { this.repositoryHttpClient = repositoryHttpClient; this.dictionaryService = dictionaryService; this.namespaceDAO = namespaceDAO; this.deserializer = new SOLRDeserializer(namespaceDAO); - this.jsonFactory = new JsonFactory(); + this.jsonFactory = new JsonFactory(); + this.compression = compression; } /** @@ -1118,20 +1137,24 @@ public class SOLRAPIClient GetRequest req = new GetRequest(url.toString()); + Map headers = new HashMap<>(); if(modifiedSince != null) { - Map headers = new HashMap(1, 1.0f); headers.put("If-Modified-Since", String.valueOf(DateUtil.formatDate(new Date(modifiedSince)))); - req.setHeaders(headers); } - + if (compression) + { + headers.put("Accept-Encoding", "gzip"); + } + req.setHeaders(headers); + Response response = repositoryHttpClient.sendRequest(req); if(response.getStatus() != Status.STATUS_NOT_MODIFIED && response.getStatus() != Status.STATUS_NO_CONTENT && response.getStatus() != Status.STATUS_OK) { throw new AlfrescoRuntimeException("GetTextContentResponse return status is " + response.getStatus()); - } - + } + return new GetTextContentResponse(response); } @@ -1481,7 +1504,8 @@ public class SOLRAPIClient private SolrApiContentStatus status; private String transformException; private String transformStatusStr; - private Long transformDuration; + private Long transformDuration; + private String contentEncoding; public GetTextContentResponse(Response response) throws IOException { @@ -1491,7 +1515,8 @@ public class SOLRAPIClient this.transformStatusStr = response.getHeader("X-Alfresco-transformStatus"); this.transformException = response.getHeader("X-Alfresco-transformException"); String tmp = response.getHeader("X-Alfresco-transformDuration"); - this.transformDuration = (tmp != null ? Long.valueOf(tmp) : null); + this.transformDuration = (tmp != null ? Long.valueOf(tmp) : null); + this.contentEncoding = response.getHeader("Content-Encoding"); setStatus(); } @@ -1557,6 +1582,11 @@ public class SOLRAPIClient public Long getTransformDuration() { return transformDuration; + } + + public String getContentEncoding() + { + return contentEncoding; } } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java index 0fbf5df7a..f467d3242 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClientFactory.java @@ -143,6 +143,7 @@ public class SOLRAPIClientFactory alfrescoHost = props.getProperty("alfresco.host", "localhost"); alfrescoPort = Integer.parseInt(props.getProperty("alfresco.port", "8080")); alfrescoPortSSL = Integer.parseInt(props.getProperty("alfresco.port.ssl", "8443")); + boolean compression = Boolean.parseBoolean(props.getProperty("solr.request.content.compress", "false")); SOLRAPIClient client = getCachedClient(alfrescoHost, alfrescoPort, alfrescoPortSSL); if (client == null) @@ -171,7 +172,7 @@ public class SOLRAPIClientFactory maxHostConnections = Integer.parseInt(props.getProperty("alfresco.maxHostConnections", "40")); socketTimeout = Integer.parseInt(props.getProperty("alfresco.socketTimeout", "60000")); - client = new SOLRAPIClient(getRepoClient(keyResourceLoader), dictionaryService, namespaceDAO); + client = new SOLRAPIClient(getRepoClient(keyResourceLoader), dictionaryService, namespaceDAO, compression); setCachedClient(alfrescoHost, alfrescoPort, alfrescoPortSSL, client); } From 37e612047bf41f48a7de9412ab3220cc8706e233 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 21 Jan 2020 16:13:59 +0100 Subject: [PATCH 106/129] Skipping transactions for DB_ID_RANGE Shard method. --- .../solr/tracker/MetadataTracker.java | 67 ++++++++++++++++++- .../alfresco/solr/client/SOLRAPIClient.java | 55 +++++++++++++-- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 6ddb64f2a..c1390bf3d 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -40,6 +40,7 @@ import org.alfresco.solr.client.Node.SolrApiNodeStatus; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.client.Transaction; import org.alfresco.solr.client.Transactions; +import org.alfresco.util.Pair; import org.apache.commons.codec.EncoderException; import org.json.JSONException; import org.slf4j.Logger; @@ -73,6 +74,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * {@link org.alfresco.solr.client.SOLRAPIClient#GET_NEXT_TX_COMMIT_TIME} */ private boolean nextTxCommitTimeServiceAvailable = false; + + /** + * Check if txInteravlCommitTimeService is available in the repository. + * This service returns the minimum and the maximum commit time for transactions in a node id range, + * so method sharding DB_ID_RANGE can skip transactions not relevant for the DB ID range. + * + * {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME} + */ + private boolean txIntervalCommitTimeServiceAvailable = false; public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -82,6 +92,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); + // Try invoking getNextTxCommitTime service try { client.getNextTxCommitTime(coreName, 0l); @@ -95,6 +106,23 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { log.error("Checking nextTxCommitTimeService failed.", e); } + + // Try invoking txIntervalCommitTime service + try + { + client.getTxIntervalCommitTime(coreName, 0l, 0l); + txIntervalCommitTimeServiceAvailable = true; + } + catch (NoSuchMethodException e) + { + log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + + "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + } + catch (Exception e) + { + log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); + } + } MetadataTracker() @@ -640,9 +668,46 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * */ - Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex()); + Long fromCommitTime = getTxFromCommitTime(txnsFound, + state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime()); log.debug("#### Check txnsFound : " + txnsFound.size()); log.debug("======= fromCommitTime: " + fromCommitTime); + + // When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction + // for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime. + // When there isn't nodes in that range, -1 is returned as commit times + if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable) + { + + DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter; + Pair commitTimes = client.getTxIntervalCommitTime(coreName, + dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); + Long shardMinCommitTime = commitTimes.getFirst(); + Long shardMaxCommitTime = commitTimes.getSecond(); + + // Node Range it's not still available in repository + if (shardMinCommitTime == -1) + { + log.debug("#### [DB_ID_RANGE] No nodes in range [" + dbIdRangeRouter.getStartRange() + "-" + + dbIdRangeRouter.getEndRange() + "] " + + "exist in the repository. Skipping metadata tracking."); + return; + } + if (fromCommitTime > shardMaxCommitTime) + { + log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [" + + dbIdRangeRouter.getStartRange() + "-" + dbIdRangeRouter.getEndRange() + "]. " + + "Skipping metadata tracking."); + return; + } + // Initial commit time for Node Range is greater than calculated from commit time + if (fromCommitTime < shardMinCommitTime) + { + log.debug("#### [DB_ID_RANGE] SKIPPING TRANSACTIONS FROM " + fromCommitTime + " TO " + + shardMinCommitTime); + fromCommitTime = shardMinCommitTime; + } + } log.debug("#### Get txn from commit time: " + fromCommitTime); transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index 55762b340..1556f7cb6 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -74,7 +73,6 @@ import org.apache.commons.codec.EncoderException; import org.apache.commons.codec.net.URLCodec; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.util.DateUtil; -import org.apache.commons.io.IOUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -106,7 +104,8 @@ public class SOLRAPIClient private static final String GET_CONTENT = "api/solr/textContent"; private static final String GET_MODEL = "api/solr/model"; private static final String GET_MODELS_DIFF = "api/solr/modelsdiff"; - private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction"; + private static final String GET_NEXT_TX_COMMIT_TIME = "api/solr/nextTransaction"; + private static final String GET_TX_INTERVAL_COMMIT_TIME = "api/solr/transactionInterval"; private static final String CHECKSUM_HEADER = "XAlfresco-modelChecksum"; @@ -1296,7 +1295,55 @@ public class SOLRAPIClient } return Long.parseLong(json.get("nextTransactionCommitTimeMs").toString()); - } + } + + /** + * Returns the minimum and the maximum commit time for transactions in a node id range. + * + * @param coreName alfresco, archive + * @param fromNodeId Id of the initial node + * @param toNodeId Id of the final node + * @return Time of the first transaction, time of the last transaction + * @throws IOException + * @throws AuthenticationException + * @throws NoSuchMethodException + */ + public Pair getTxIntervalCommitTime(String coreName, Long fromNodeId, Long toNodeId) + throws AuthenticationException, IOException, NoSuchMethodException + { + StringBuilder url = new StringBuilder(GET_TX_INTERVAL_COMMIT_TIME); + url.append("?").append("fromNodeId").append("=").append(fromNodeId); + url.append("&").append("toNodeId").append("=").append(toNodeId); + GetRequest get = new GetRequest(url.toString()); + Response response = null; + JSONObject json = null; + try + { + response = repositoryHttpClient.sendRequest(get); + if (response.getStatus() != HttpStatus.SC_OK) + { + throw new NoSuchMethodException(coreName + " - GetTxIntervalCommitTime return status is " + + response.getStatus() + " when invoking " + url); + } + + Reader reader = new BufferedReader(new InputStreamReader(response.getContentAsStream(), "UTF-8")); + json = new JSONObject(new JSONTokener(reader)); + } + finally + { + if (response != null) + { + response.release(); + } + } + if (log.isDebugEnabled()) + { + log.debug(json.toString()); + } + + return new Pair(Long.parseLong(json.get("minTransactionCommitTimeMs").toString()), + Long.parseLong(json.get("maxTransactionCommitTimeMs").toString())); + } /* * type conversions from serialized JSON values to SOLR-consumable objects From 58c1526963ad298dd0b95efa28abe20f101b4ed2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2020 22:13:44 +0000 Subject: [PATCH 107/129] Bump alfresco-data-model from 8.75 to 8.87 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.75 to 8.87. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.75...8.87) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index bfd246059..11bd9ea0c 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.75 + 8.87 2.10.2 From 2882b8edc23a6f5b641329fdda22eecfdff128d2 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Wed, 22 Jan 2020 10:37:37 +0100 Subject: [PATCH 108/129] Add Get Content compress option to Yeoman Generator. --- .../generator-alfresco-docker-compose/README.md | 3 ++- .../generators/app/index.js | 12 ++++++++++-- .../app/templates/6.2/docker-compose-ce.yml | 3 +++ .../app/templates/images/alfresco/Dockerfile | 16 +++++++++++++++- .../app/templates/images/search/Dockerfile | 8 ++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/e2e-test/generator-alfresco-docker-compose/README.md b/e2e-test/generator-alfresco-docker-compose/README.md index 986904543..0caf4d33d 100644 --- a/e2e-test/generator-alfresco-docker-compose/README.md +++ b/e2e-test/generator-alfresco-docker-compose/README.md @@ -111,7 +111,8 @@ When using Community, some different options can be combined: ? Would you like to use HTTP or mTLS for Alfresco-SOLR communication? http ? Would you like to use HTTP or HTTPs for Web Proxy? http ? Would you like to protect the access to SOLR REST API? Yes -? Would you like to use a SOLR Replication (2 nodes in master-slave)? No +? Would you like to use a SOLR Replication? No +? Would you like to compress Get Content responses? No ``` **Note** that when choosing *mTLS* or *HTTPs*, default certificates, truststores and keystores are provided for testing purposes. If you are planning to use this Docker Compose template for real environments, replace these cryptographic stores with another generated by yourself to increase the security of your system. diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js index ac5b2f4a2..03905232a 100644 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js @@ -56,7 +56,7 @@ module.exports = class extends Generator { type: 'confirm', name: 'protectSolr', message: 'Would you like to protect the access to SOLR REST API?', - default: 'true' + default: true }, { whenFunction: response => response.httpMode == 'http', @@ -69,6 +69,13 @@ module.exports = class extends Generator { { name: "Yes - two nodes in a master-master configuration", value: "master-master" } ] }, + { + whenFunction: response => response.acsVersion == '6.2', + type: 'confirm', + name: 'gzip', + message: 'Would you like to compress Get Content responses?', + default: false + }, // Enterprise only options { whenFunction: response => response.alfrescoVersion == 'enterprise' && !response.replication, @@ -208,7 +215,8 @@ module.exports = class extends Generator { searchPath: searchBasePath, zeppelin: (this.props.zeppelin ? "true" : "false"), sharding: (this.props.sharding ? "true" : "false"), - shardingMethod: (this.props.shardingMethod) + shardingMethod: (this.props.shardingMethod), + gzip: (this.props.gzip ? "true" : "false") } ); diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml index 451e57945..7c137e4a0 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml @@ -13,6 +13,7 @@ services: TRUSTSTORE_PASS: kT9X6oe68t KEYSTORE_TYPE: JCEKS KEYSTORE_PASS: kT9X6oe68t <% } %> + COMPRESS_CONTENT: "<%=gzip%>" mem_limit: 1800m environment: JAVA_OPTS : " @@ -66,6 +67,7 @@ services: KEYSTORE_TYPE: JCEKS <% } %> <% if (replication) { %> ENABLE_MASTER: "true" ENABLE_SLAVE: "false" <% } %> + COMPRESS_CONTENT: "<%=gzip%>" mem_limit: 1200m environment: #Solr needs to know how to register itself with Alfresco @@ -108,6 +110,7 @@ services: ENABLE_MASTER: <% if (replication == 'master-master') { %>"true"<% } else { %>"false"<% } %> ENABLE_SLAVE: <% if (replication == 'master-master') { %>"false"<% } else { %>"true"<% } %> MASTER_HOST: solr6 <% } %> + COMPRESS_CONTENT: "<%=gzip%>" mem_limit: 1200m environment: #Solr needs to know how to register itself with Alfresco diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile index a5595c7ac..87091db7a 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile @@ -40,13 +40,27 @@ RUN if [ "$SOLR_COMMS" == "https" ] ; then \ [[:space:]]\+<\/Engine>/\n\ <\/Engine>\n\ \n\ <\/Connector>/g" ${TOMCAT_DIR}/conf/server.xml; \ fi +# GZIP COMPRESSION +ARG COMPRESS_CONTENT +ENV COMPRESS_CONTENT $COMPRESS_CONTENT +RUN if [ "$COMPRESS_CONTENT" == "true" ] ; then \ + sed -i "s/\ +[[:space:]]\+connectionTimeout=\"20000\"/\n\ + connectionTimeout=\"20000\"\n\ + compression=\"on\"\n\ + compressionMinSize=\"1\"\n\ + /g" ${TOMCAT_DIR}/conf/server.xml; \ + fi + + # Copy custom content model to deployment folder COPY model/* $TOMCAT_DIR/shared/classes/alfresco/extension/ diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile index 163f75682..0ff7861ba 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile @@ -125,6 +125,14 @@ RUN if [ "$ENABLE_SHARDING" == "true" ] ; then \ fi; \ fi +# GZIP COMPRESSION +ARG COMPRESS_CONTENT +ENV COMPRESS_CONTENT $COMPRESS_CONTENT +RUN if [ "$COMPRESS_CONTENT" == "true" ] ; then \ + sed -i '/^bash.*/i sed -i "'"s/solr.request.content.compress=false/solr.request.content.compress=true/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + ${DIST_DIR}/solr/bin/search_config_setup.sh; \ +fi + # Useless for 'none'/'http' communications with Alfresco RUN mkdir ${DIST_DIR}/keystore \ && chown -R solr:solr ${DIST_DIR}/keystore From 91cb2f2706fc9a626aa3c98b3e3e21d659ca396c Mon Sep 17 00:00:00 2001 From: Keerat Date: Wed, 22 Jan 2020 11:18:07 +0000 Subject: [PATCH 109/129] SEARCH-1966 master slave test --- .../search/functional/searchServices/search/ShardInfoTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java index 74fbcf971..e8cf8b9a0 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java @@ -101,6 +101,7 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest RestShardInfoModel model = shardInfoModel.getModel(); assertEquals(model.getTemplate(), "rerank"); assertEquals(model.getShardMethod(), "DB_ID"); + assertEquals(model.getMode(), "MIXED"); assertTrue(model.getHasContent()); assertTrue(stores.contains(model.getStores())); From 81c8c1f51f78df11e34a3b143bc651ee731893a8 Mon Sep 17 00:00:00 2001 From: Keerat Date: Wed, 22 Jan 2020 12:03:37 +0000 Subject: [PATCH 110/129] SEARCH-1966 master slave test --- .../search/functional/searchServices/search/ShardInfoTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java index e8cf8b9a0..4423f3d1e 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java @@ -118,6 +118,7 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest assertTrue(baseUrls.contains(instance.getBaseUrl())); assertEquals(instance.getState(), "ACTIVE"); + assertEquals(instance.getMode(), "MIXED"); } } From f708f89bb22fb6a66f6321293ea7fcc9e95d2eba Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 22 Jan 2020 15:01:07 +0000 Subject: [PATCH 111/129] SEARCH-2067 Try fixing integration test order as alphabetical. Some of our integration tests fail if they are run after other tests. --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 9af7a2537..f21eb44c8 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,9 @@ + + alphabetical + From 3854bf20601a48cd2556cba0fa415733c764419d Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Wed, 22 Jan 2020 18:19:32 +0100 Subject: [PATCH 112/129] Avoid checking repo services when using MetadataTracker from Integration Tests (locally using JUnit) --- .../solr/lifecycle/SolrCoreLoadListener.java | 2 +- .../solr/tracker/MetadataTracker.java | 79 ++++++++++++------- 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index 66ed18b9e..fc7597a0f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -251,7 +251,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener MetadataTracker metadataTracker = registerAndSchedule( - new MetadataTracker(true, props, repositoryClient, core.getName(), srv), + new MetadataTracker(true, props, repositoryClient, core.getName(), srv, true), core, props, trackerRegistry, diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index c1390bf3d..4821278b4 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -85,42 +85,63 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker private boolean txIntervalCommitTimeServiceAvailable = false; public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, - InformationServer informationServer) + InformationServer informationServer) + { + this(isMaster, p, client, coreName, informationServer, false); + } + + /** + * MetadataTracker constructor + * + * @param isMaster is true if SOLR instance is master, false otherwise + * @param p includes SOLR core properties (from environment variables and properties file) + * @param client Alfresco Repository http client + * @param coreName Name of the SOLR Core (alfresco, archive) + * @param informationServer SOLR Information Server + * @param checkRepoServicesAvailability is true if Repo Services availability needs to be checked + */ + public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, + InformationServer informationServer, boolean checkRepoServicesAvailability) { super(isMaster, p, client, coreName, informationServer, Tracker.Type.METADATA); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); - // Try invoking getNextTxCommitTime service - try + // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. + // As these services are available from ACS 6.2 + if (checkRepoServicesAvailability) { - client.getNextTxCommitTime(coreName, 0l); - nextTxCommitTimeServiceAvailable = true; - } - catch (NoSuchMethodException e) - { - log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); - } - catch (Exception e) - { - log.error("Checking nextTxCommitTimeService failed.", e); - } - - // Try invoking txIntervalCommitTime service - try - { - client.getTxIntervalCommitTime(coreName, 0l, 0l); - txIntervalCommitTimeServiceAvailable = true; - } - catch (NoSuchMethodException e) - { - log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " - + "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); - } - catch (Exception e) - { - log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); + // Try invoking getNextTxCommitTime service + try + { + client.getNextTxCommitTime(coreName, 0l); + nextTxCommitTimeServiceAvailable = true; + } + catch (NoSuchMethodException e) + { + log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + } + catch (Exception e) + { + log.error("Checking nextTxCommitTimeService failed.", e); + } + + // Try invoking txIntervalCommitTime service + try + { + client.getTxIntervalCommitTime(coreName, 0l, 0l); + txIntervalCommitTimeServiceAvailable = true; + } + catch (NoSuchMethodException e) + { + log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + + "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + } + catch (Exception e) + { + log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); + } } } From e7783c734c8f141dc19d43f68febe2413dde4582 Mon Sep 17 00:00:00 2001 From: Keerat Date: Thu, 23 Jan 2020 08:48:52 +0000 Subject: [PATCH 113/129] SEARCH-1966 Master slave infastructure --- .../search/functional/searchServices/search/ShardInfoTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java index 4423f3d1e..e8cf8b9a0 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java @@ -118,7 +118,6 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest assertTrue(baseUrls.contains(instance.getBaseUrl())); assertEquals(instance.getState(), "ACTIVE"); - assertEquals(instance.getMode(), "MIXED"); } } From 1e9e3fabc4f74c78288852e76e7b73f9f736a210 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Jan 2020 08:56:09 +0000 Subject: [PATCH 114/129] SEARCH-2046 Dependency management for alfresco-data-model upgrade. Upgrade Spring, remove a few libraries and override version of commons-lang3 and xpp3 as these are no longer provided by alfresco-data-model. --- search-services/alfresco-search/pom.xml | 10 ++++++++++ search-services/packaging/pom.xml | 3 ++- .../src/main/resources/licenses/notice.txt | 20 ++++++++----------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 1bffd3a2b..2e37b62de 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -96,6 +96,11 @@ 2.3.2 + + org.apache.commons + commons-lang3 + 3.9 + org.apache.cxf cxf-core @@ -146,6 +151,11 @@ cxf-rt-wsdl ${cxf.version} + + xpp3 + xpp3 + 1.1.4c + diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 1ed5774f9..6f3c51e63 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -111,7 +111,8 @@ ${project.version} libs ${project.build.directory}/solr-libs - **/jackson-dataformat-smile-*.jar,**/asm-3.3.1.jar,**/jackson-core-asl-*.jar,**/jackson-mapper-asl-*.jar,**/dom4j-1.6.1.jar,**/annotations-1.0.0.jar,**/woodstox-core-asl-4.4.1.jar + **/jackson-dataformat-smile-*.jar,**/asm-3.3.1.jar,**/jackson-core-asl-*.jar,**/jackson-mapper-asl-*.jar,**/dom4j-1.6.1.jar, + **/annotations-1.0.0.jar,**/spring-context-support-*.jar,**/spring-web-*.jar,**/woodstox-core-asl-4.4.1.jar diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 3ff7a566b..5e61057de 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -35,7 +35,6 @@ xml-resolver-1.2.jar https://github.com/FasterXML/jackson neethi-3.1.1.jar http://ws.apache.org/commons/neethi/ commons-logging-1.2.jar http://jakarta.apache.org/commons/ commons-lang3-3.9.jar http://jakarta.apache.org/commons/ -mybatis-3.3.0.jar http://www.mybatis.org/ chemistry-opencmis-commons-impl-1.1.0.jar http://chemistry.apache.org/ chemistry-opencmis-commons-api-1.1.0.jar http://chemistry.apache.org/ xmlschema-core-2.2.3.jar http://ws.apache.org/commons/XmlSchema/ @@ -50,7 +49,6 @@ cxf-rt-transports-http-3.2.5.jar https://cxf.apache.org/ cxf-rt-ws-addr-3.2.5.jar https://cxf.apache.org/ cxf-rt-ws-policy-3.2.5.jar https://cxf.apache.org/ cxf-rt-wsdl-3.2.5.jar https://cxf.apache.org/ -mybatis-spring-1.2.5.jar http://www.mybatis.org/ chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/ chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/ quartz-2.3.2.jar http://quartz-scheduler.org/ @@ -58,16 +56,14 @@ jackson-core-2.10.2.jar https://github.com/FasterXML/jackson jackson-annotations-2.10.2.jar https://github.com/FasterXML/jackson jackson-databind-2.10.2.jar https://github.com/FasterXML/jackson commons-httpclient-3.1-HTTPCLIENT-1265.jar http://jakarta.apache.org/commons/ -spring-aop-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-beans-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-context-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-context-support-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-core-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-expression-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-jdbc-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-orm-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-tx-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ -spring-web-5.2.2.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-aop-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-beans-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-context-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-core-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-expression-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-jdbc-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-orm-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ +spring-tx-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/ xercesImpl-2.12.0-alfresco-patched-20191004.jar http://xerces.apache.org/xerces2-j guessencoding-1.4.jar http://docs.codehaus.org/display/GUESSENC/ xml-apis-1.4.01.jar https://github.com/FasterXML/jackson From 3de4b6a17b0bc959916acb9f1e4ab61df4475326 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Jan 2020 11:53:04 +0000 Subject: [PATCH 115/129] Remove outdated comment. --- search-services/packaging/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index d349ced14..6292f10b6 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -60,7 +60,7 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.5.0 + 1.5.0 unpack-solr-war From d98bf928c39f6a04c81a0f8224a7e18b574393a4 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Jan 2020 16:10:31 +0000 Subject: [PATCH 116/129] SEARCH-1949 Add config option to disable cascade tracking. --- .../src/main/resources/solr/instance/conf/shared.properties | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties index e78b8ee32..63bc6bfe2 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties @@ -30,4 +30,7 @@ alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lock # alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content # alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext -alfresco.model.tracker.cron=0/10 * * * * ? * \ No newline at end of file +alfresco.model.tracker.cron=0/10 * * * * ? * + +# Whether path queries are enabled. +alfresco.cascade.tracker.enabled=true From 6268b22a2f10b07f062d8d7dc49c135864c35658 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 23 Jan 2020 16:23:14 +0000 Subject: [PATCH 117/129] SEARCH-1949 Rename some integration tests as we can still have cascading without the tracker. --- ...rackerIntegrationTest.java => CascadingIntegrationTest.java} | 2 +- .../solr/tracker/{CascadeTrackerIT.java => CascadingIT.java} | 2 +- ...stributedCascadeTrackerIT.java => DistributedCascadeIT.java} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/{CascadingTrackerIntegrationTest.java => CascadingIntegrationTest.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{CascadeTrackerIT.java => CascadingIT.java} (99%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{DistributedCascadeTrackerIT.java => DistributedCascadeIT.java} (98%) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java similarity index 99% rename from e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java rename to e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java index 3d66f1922..a61e2dc63 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java @@ -31,7 +31,7 @@ import static org.testng.Assert.assertTrue; * @author Alessandro Benedetti * @author Meenal Bhave */ -public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest +public class CascadingIntegrationTest extends AbstractE2EFunctionalTest { @Autowired protected DataContent dataContent; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java similarity index 99% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java index 00c34cfa4..7d1e145a9 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadeTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/CascadingIT.java @@ -52,7 +52,7 @@ import java.util.List; @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) @SolrTestCaseJ4.SuppressSSL -public class CascadeTrackerIT extends AbstractAlfrescoSolrIT +public class CascadingIT extends AbstractAlfrescoSolrIT { private static long MAX_WAIT_TIME = 80000; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeIT.java index 6e5fbe39f..3d1415bb4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeIT.java @@ -55,7 +55,7 @@ import java.util.Properties; */ @SolrTestCaseJ4.SuppressSSL @LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) -public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT +public class DistributedCascadeIT extends AbstractAlfrescoDistributedIT { private Node parentFolder; private NodeMetaData parentFolderMetadata; From 0d7f979d5f3e0eeeadabe520cf41b271aab16a7a Mon Sep 17 00:00:00 2001 From: mbhave Date: Thu, 23 Jan 2020 17:32:00 +0000 Subject: [PATCH 118/129] Search-1966: cmis tests amended to wait for indexing to allow slave to sync with master --- .../functional/AbstractE2EFunctionalTest.java | 2 +- .../cmis/AbstractCmisE2ETest.java | 4 +- .../cmis/SolrSearchByAspectTests.java | 4 +- .../cmis/SolrSearchByIdTests.java | 6 +- .../cmis/SolrSearchByPathTests.java | 5 +- .../cmis/SolrSearchByPropertyTests.java | 4 +- .../cmis/SolrSearchInFolderTests.java | 4 +- .../cmis/SolrSearchInTreeTests.java | 5 +- .../cmis/SolrSearchScoreQueryTests.java | 147 +++++++----------- 9 files changed, 82 insertions(+), 99 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index 8073d003d..eec4195e2 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -55,7 +55,7 @@ import org.testng.annotations.BeforeSuite; public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests { /** The number of retries that a query will be tried before giving up. */ - private static final int SEARCH_MAX_ATTEMPTS = 6; + protected static final int SEARCH_MAX_ATTEMPTS = 6; private static final Logger LOGGER = LogFactory.getLogger(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java index 3dc7ed8c3..f096c15ff 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java @@ -61,7 +61,7 @@ public abstract class AbstractCmisE2ETest extends AbstractE2EFunctionalTest protected boolean waitForIndexing(String query, long expectedCountResults) { - for (int searchCount = 1; searchCount <= 3; searchCount++) + for (int searchCount = 1; searchCount <= SEARCH_MAX_ATTEMPTS; searchCount++) { try @@ -71,7 +71,7 @@ public abstract class AbstractCmisE2ETest extends AbstractE2EFunctionalTest } catch (AssertionError ae) { - LOGGER.debug(ae.toString()); + LOGGER.info(String.format("WaitForIndexing in Progress: %s", ae.toString())); } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java index 8e0e0eb08..5228c38e6 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java @@ -7,6 +7,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; +import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -135,6 +136,7 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest .replace("NODE_REF[f1]", tasFolder1.getNodeRef()) .replace("NODE_REF[s1]", siteDoclibNodeRef); - cmisApi.authenticateUser(testUser).withQuery(currentQuery).assertResultsCount().equals(query.getResults()); + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java index 5d1026e29..8d10955fd 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java @@ -7,6 +7,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; +import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -87,7 +88,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") @XMLDataConfig(file = "src/test/resources/testdata/search-by-id.xml") - public void executeSearchByAspect(QueryModel query) throws Exception + public void executeSearchById(QueryModel query) throws Exception { String currentQuery = query.getValue() .replace("NODE_REF[siteId]", siteDoclibNodeRef) @@ -96,6 +97,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest .replace("NODE_REF[f1]", tasFolder1.getNodeRef()) .replace("NODE_REF[f1-1]", tasSubFolder1.getNodeRef()); - cmisApi.authenticateUser(testUser).withQuery(currentQuery).assertResultsCount().equals(query.getResults()); + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java index c1e0ae1b9..1c86b2912 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java @@ -7,6 +7,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.QueryModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -51,6 +52,8 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest @XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml") public void executeSearchByPathQueries(QueryModel query) { - cmisApi.withQuery(query.getValue()).assertResultsCount().equals(query.getResults()); + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(query.getValue(), query.getResults()), String.format("Result count not as expected for query: %s", query.getValue())); + } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java index d9f7638bd..9b6ad81e7 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java @@ -10,6 +10,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; +import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -104,7 +105,8 @@ public class SolrSearchByPropertyTests extends AbstractCmisE2ETest .addProperty("tas:IntPropertyC", 2223)); // wait for solr index - Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); + cmisApi.authenticateUser(testUser); + waitForIndexing("SELECT * FROM tas:document where cmis:name = 'testc3.txt'", 1); } @Test diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java index 17647a6ed..bc903f30d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java @@ -7,6 +7,7 @@ import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -55,6 +56,7 @@ public class SolrSearchInFolderTests extends AbstractCmisE2ETest public void executeCMISQuery(QueryModel query) throws Exception { String currentQuery = String.format(query.getValue(), parentFolder.getNodeRef()); - cmisApi.withQuery(currentQuery).assertResultsCount().equals(query.getResults()); + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java index 3fd68cfa4..cf6647e86 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java @@ -7,6 +7,7 @@ import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -59,7 +60,7 @@ public class SolrSearchInTreeTests extends AbstractCmisE2ETest public void executeCMISQuery(QueryModel query) throws Exception { String currentQuery = String.format(query.getValue(), parentFolder.getNodeRef()); - cmisApi.withQuery(currentQuery) - .assertResultsCount().equals(query.getResults()); + cmisApi.authenticateUser(testUser); + Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java index 85b597fea..847c5bc1b 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java @@ -9,6 +9,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -54,8 +55,8 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest this.testData = testData; this.testData.createUsers(dataUser); this.testData.createSitesStructure(dataSite, dataContent, dataUser); - cmisApi.authenticateUser(dataUser.getCurrentUser()); - + testUser = dataUser.getCurrentUser(); + cmisApi.authenticateUser(testUser); } /** @@ -70,68 +71,50 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest + "FROM cmis:document " + "WHERE CONTAINS('Quidditch') " + "ORDER BY orderCriteria"; - - if (waitForIndexing(query, 3)) - { - cmisApi - .withQuery(query) - .assertColumnIsOrdered().isOrderedAsc("orderCriteria"); - } - else - { - throw new AssertionError("Wait for indexing has failed!"); - } - + + Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query)); + + cmisApi.withQuery(query).assertColumnIsOrdered().isOrderedAsc("orderCriteria"); + } - - /** - * Verify that results are inverse ordered - * @throws Exception - */ - @Test(dependsOnMethods = "prepareDataForScoreSearch") + + /** + * Verify that results are inverse ordered + * + * @throws Exception + */ + @Test(dependsOnMethods = "prepareDataForScoreSearch") public void scoreQueryOrderedDesc() throws Exception { - String query = "SELECT cmis:objectId, SCORE() AS orderCriteria " + String query = "SELECT cmis:objectId, SCORE() AS orderCriteria " + "FROM cmis:document " + "WHERE CONTAINS('Quidditch') " + "ORDER BY orderCriteria DESC"; - - if (waitForIndexing(query, 3)) - { - cmisApi - .withQuery(query).assertColumnIsOrdered().isOrderedDesc("orderCriteria"); - } - else - { - throw new AssertionError("Wait for indexing has failed!"); - } - + + Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query)); + + cmisApi.withQuery(query).assertColumnIsOrdered().isOrderedDesc("orderCriteria"); + } - - /** - * Verify that all SCORE results are between 0 and 1 - * @throws Exception - */ - @Test(groups = { TestGroup.ACS_62n }, dependsOnMethods = "prepareDataForScoreSearch") + + /** + * Verify that all SCORE results are between 0 and 1 + * + * @throws Exception + */ + @Test(groups = { TestGroup.ACS_62n }, dependsOnMethods = "prepareDataForScoreSearch") public void scoreQueryInRange() throws Exception { - - String query = "SELECT cmis:objectId, SCORE() " - + "FROM cmis:document " - + "WHERE CONTAINS('Quidditch')"; - - if (waitForIndexing(query, 3)) - { - cmisApi - .withQuery(query) - .assertColumnValuesRange().isReturningValuesInRange("SEARCH_SCORE", BigDecimal.ZERO, BigDecimal.ONE); - } - else - { - throw new AssertionError("Wait for indexing has failed!"); - } + + String query = "SELECT cmis:objectId, SCORE() " + + "FROM cmis:document " + + "WHERE CONTAINS('Quidditch')"; + Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query)); + + cmisApi.withQuery(query).assertColumnValuesRange().isReturningValuesInRange("SEARCH_SCORE", BigDecimal.ZERO, BigDecimal.ONE); + } /** @@ -143,45 +126,33 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest { String query = "SELECT cmis:objectId, SCORE() AS orderCriteria " - + "FROM cmis:document " - + "WHERE CONTAINS('Quidditch')"; - - if (waitForIndexing(query, 3)) - { - cmisApi - .withQuery(query) - .assertColumnValuesRange().isReturningValuesInRange("orderCriteria", BigDecimal.ZERO, BigDecimal.ONE); - } - else - { - throw new AssertionError("Wait for indexing has failed!"); - } - + + "FROM cmis:document " + + "WHERE CONTAINS('Quidditch')"; + + Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query)); + + cmisApi.withQuery(query).assertColumnValuesRange().isReturningValuesInRange("orderCriteria", BigDecimal.ZERO, BigDecimal.ONE); + } - /** - * Verify that SCORE is valid name for an alias - * Currently only supported with double quotes - * @throws Exception - */ - @Test(dependsOnMethods = "prepareDataForScoreSearch") + /** + * Verify that SCORE is valid name for an alias + * Currently only supported with double quotes + * + * @throws Exception + */ + @Test(dependsOnMethods = "prepareDataForScoreSearch") public void scoreQueryScoreAsAlias() throws Exception { - - String query = "SELECT cmis:objectId, SCORE() AS \"score\" " - + "FROM cmis:document " - + "WHERE CONTAINS('Quidditch')"; - - if (waitForIndexing(query, 3)) - { - cmisApi - .withQuery(query).assertResultsCount().equals(3); - } - else - { - throw new AssertionError("Wait for indexing has failed!"); - } - + + String query = "SELECT cmis:objectId, SCORE() AS \"score\" " + + "FROM cmis:document " + + "WHERE CONTAINS('Quidditch')"; + + Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query)); + + cmisApi.withQuery(query).assertResultsCount().equals(3); + } } From e97aea97bf77de5e2b91ceff09b0fbf8dcff6a9f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2020 22:12:14 +0000 Subject: [PATCH 119/129] Bump utility from 3.0.18 to 3.0.19 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.18 to 3.0.19. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.18...utility-3.0.19) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 1e48fef17..628c981b8 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -14,7 +14,7 @@ 1.28 1.26 1.13 - 3.0.18 + 3.0.19 3.3.0 src/test/resources/SearchSuite.xml From 7b0b60d56f495e99f990042262319ad20dd70d7f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2020 22:12:48 +0000 Subject: [PATCH 120/129] Bump cxf.version from 3.2.5 to 3.2.12 in /search-services Bumps `cxf.version` from 3.2.5 to 3.2.12. Updates `cxf-core` from 3.2.5 to 3.2.12 Updates `cxf-rt-bindings-soap` from 3.2.5 to 3.2.12 Updates `cxf-rt-bindings-xml` from 3.2.5 to 3.2.12 Updates `cxf-rt-databinding-jaxb` from 3.2.5 to 3.2.12 Updates `cxf-rt-frontend-jaxws` from 3.2.5 to 3.2.12 Updates `cxf-rt-frontend-simple` from 3.2.5 to 3.2.12 Updates `cxf-rt-transports-http` from 3.2.5 to 3.2.12 Updates `cxf-rt-ws-addr` from 3.2.5 to 3.2.12 Updates `cxf-rt-ws-policy` from 3.2.5 to 3.2.12 Updates `cxf-rt-wsdl` from 3.2.5 to 3.2.12 Signed-off-by: dependabot-preview[bot] --- search-services/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/pom.xml b/search-services/pom.xml index 9a316f644..53e3c12de 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -15,7 +15,7 @@ Alfresco Solr Search parent 1.7.30 - 3.2.5 + 3.2.12 alfresco-solrclient-lib From c5d6777a721ebc1aaa4f01da5a86e0e05147a86a Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 08:19:54 +0000 Subject: [PATCH 121/129] Update license declaration for cxf 3.2.12. --- .../src/main/resources/licenses/notice.txt | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/search-services/packaging/src/main/resources/licenses/notice.txt b/search-services/packaging/src/main/resources/licenses/notice.txt index 5e61057de..b3eb46870 100644 --- a/search-services/packaging/src/main/resources/licenses/notice.txt +++ b/search-services/packaging/src/main/resources/licenses/notice.txt @@ -37,18 +37,18 @@ commons-logging-1.2.jar http://jakarta.apache.org/commons/ commons-lang3-3.9.jar http://jakarta.apache.org/commons/ chemistry-opencmis-commons-impl-1.1.0.jar http://chemistry.apache.org/ chemistry-opencmis-commons-api-1.1.0.jar http://chemistry.apache.org/ -xmlschema-core-2.2.3.jar http://ws.apache.org/commons/XmlSchema/ +xmlschema-core-2.2.5.jar http://ws.apache.org/commons/XmlSchema/ HikariCP-java7-2.4.13.jar https://github.com/brettwooldridge/HikariCP -cxf-core-3.2.5.jar https://cxf.apache.org/ -cxf-rt-bindings-soap-3.2.5.jar https://cxf.apache.org/ -cxf-rt-bindings-xml-3.2.5.jar https://cxf.apache.org/ -cxf-rt-databinding-jaxb-3.2.5.jar https://cxf.apache.org/ -cxf-rt-frontend-jaxws-3.2.5.jar https://cxf.apache.org/ -cxf-rt-frontend-simple-3.2.5.jar https://cxf.apache.org/ -cxf-rt-transports-http-3.2.5.jar https://cxf.apache.org/ -cxf-rt-ws-addr-3.2.5.jar https://cxf.apache.org/ -cxf-rt-ws-policy-3.2.5.jar https://cxf.apache.org/ -cxf-rt-wsdl-3.2.5.jar https://cxf.apache.org/ +cxf-core-3.2.12.jar https://cxf.apache.org/ +cxf-rt-bindings-soap-3.2.12.jar https://cxf.apache.org/ +cxf-rt-bindings-xml-3.2.12.jar https://cxf.apache.org/ +cxf-rt-databinding-jaxb-3.2.12.jar https://cxf.apache.org/ +cxf-rt-frontend-jaxws-3.2.12.jar https://cxf.apache.org/ +cxf-rt-frontend-simple-3.2.12.jar https://cxf.apache.org/ +cxf-rt-transports-http-3.2.12.jar https://cxf.apache.org/ +cxf-rt-ws-addr-3.2.12.jar https://cxf.apache.org/ +cxf-rt-ws-policy-3.2.12.jar https://cxf.apache.org/ +cxf-rt-wsdl-3.2.12.jar https://cxf.apache.org/ chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/ chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/ quartz-2.3.2.jar http://quartz-scheduler.org/ From 8eaf47f9237a892f511c52b6b690f2fa4fe5cd46 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Fri, 24 Jan 2020 10:45:11 +0100 Subject: [PATCH 122/129] Skip getting Child Ids when indexing metadata. --- .../java/org/alfresco/solr/SolrInformationServer.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 182067271..f46db9667 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1818,6 +1818,12 @@ public class SolrInformationServer implements InformationServer { NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); nmdp.setNodeIds(unknownNodeIds); + // When deleting nodes, no additional information is required + nmdp.setIncludeChildIds(false); + nmdp.setIncludeChildAssociations(false); + nmdp.setIncludeAspects(false); + nmdp.setIncludePaths(false); + nmdp.setIncludeParentAssociations(false); nodeMetaDatas.addAll(repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE)); } @@ -1859,10 +1865,11 @@ public class SolrInformationServer implements InformationServer nodeIds.addAll(unknownNodeIds); nodeIds.addAll(shardUpdatedNodeIds); nmdp.setNodeIds(nodeIds); + nmdp.setIncludeChildIds(false); // Fetches bulk metadata List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); - + NEXT_NODE: for (NodeMetaData nodeMetaData : nodeMetaDatas) { From 943fafcca383a995db221805fed2cd9036ec66e2 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Fri, 24 Jan 2020 14:59:50 +0100 Subject: [PATCH 123/129] Child associations are not indexed as well. --- .../src/main/java/org/alfresco/solr/SolrInformationServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index f46db9667..205c3bc90 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -1866,6 +1866,7 @@ public class SolrInformationServer implements InformationServer nodeIds.addAll(shardUpdatedNodeIds); nmdp.setNodeIds(nodeIds); nmdp.setIncludeChildIds(false); + nmdp.setIncludeChildAssociations(false); // Fetches bulk metadata List nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); From 6ceabbaaac524ab05b15145b781fc5884975b658 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 09:25:12 +0000 Subject: [PATCH 124/129] SEARCH-1949 Add helper method to check if cascade tracking is enabled. --- .../alfresco/solr/AlfrescoSolrDataModel.java | 2 +- .../alfresco/solr/SolrInformationServer.java | 165 ++++++++++-------- 2 files changed, 94 insertions(+), 73 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index c28d7ae8b..50b6d9563 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -523,7 +523,7 @@ public class AlfrescoSolrDataModel implements QueryConstants } catch (IOException e) { - log.info("Failed to read shared properties fat " + propertiesFile.getAbsolutePath()); + log.info("Failed to read shared properties at " + propertiesFile.getAbsolutePath()); } return props; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 182067271..fe94569ff 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -205,10 +205,7 @@ public class SolrInformationServer implements InformationServer public static final String AND = " AND "; public static final String OR = " OR "; - //public static final String REQUEST_HANDLER_ALFRESCO_FULL_TEXT_SEARCH = "/afts"; private static final String REQUEST_HANDLER_NATIVE = "/native"; - //public static final String REQUEST_HANDLER_ALFRESCO = "/alfresco"; - //public static final String REQUEST_HANDLER_SELECT = "/select"; static final String REQUEST_HANDLER_GET = "/get"; private static final String RESPONSE_DEFAULT_IDS = "response"; static final String RESPONSE_DEFAULT_ID = "doc"; @@ -441,7 +438,7 @@ public class SolrInformationServer implements InformationServer dataModel = AlfrescoSolrDataModel.getInstance(); contentStreamLimit = Integer.parseInt(p.getProperty("alfresco.contentStreamLimit", "10000000")); - + // build base URL - host and port have to come from configuration. props = AlfrescoSolrDataModel.getCommonConfig(); hostName = ConfigUtil.locateProperty(SOLR_HOST, props.getProperty(SOLR_HOST)); @@ -456,6 +453,17 @@ public class SolrInformationServer implements InformationServer return this.adminHandler; } + /** + * Check if cascade tracking is enabled. + * + * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). + */ + private boolean cascadeTrackingEnabled() + { + String cascadeTrackerEnabledProp = ofNullable((String) props.get("alfresco.cascade.tracker.enabled")).orElse("true"); + return Boolean.valueOf(cascadeTrackerEnabledProp); + } + @Override public synchronized void initSkippingDescendantDocs() { @@ -507,7 +515,7 @@ public class SolrInformationServer implements InformationServer report.add("Node count with FTSStatus New", newCount); } } - + @Override public void afterInitModels() { @@ -520,7 +528,7 @@ public class SolrInformationServer implements InformationServer String query = FIELD_ACLID + ":" + aclid + AND + FIELD_DOC_TYPE + ":" + DOC_TYPE_ACL; long count = this.getDocListSize(query); aclReport.setIndexedAclDocCount(count); - + // TODO Could add INACLTXID later, but would need acl change set id. return aclReport; } @@ -778,7 +786,7 @@ public class SolrInformationServer implements InformationServer long count = this.getDocListSize(query); nodeReport.setIndexedNodeDocCount(count); } - + @Override public void commit() throws IOException { @@ -891,26 +899,26 @@ public class SolrInformationServer implements InformationServer return searcherOpened; } - + @Override public void deleteByAclChangeSetId(Long aclChangeSetId) throws IOException { deleteById(FIELD_INACLTXID, aclChangeSetId); } - + @Override public void deleteByAclId(Long aclId) throws IOException { isIdIndexCache.clear(); deleteById(FIELD_ACLID, aclId); } - + @Override public void deleteByNodeId(Long nodeId) throws IOException { deleteById(FIELD_DBID, nodeId); } - + @Override public void deleteByTransactionId(Long transactionId) throws IOException { @@ -923,7 +931,7 @@ public class SolrInformationServer implements InformationServer { return this.dataModel.getAlfrescoModels(); } - + @SuppressWarnings({ "unchecked", "rawtypes" }) @Override public Iterable> getCoreStats() throws IOException @@ -1047,7 +1055,7 @@ public class SolrInformationServer implements InformationServer return coreSummary; } - + @Override public DictionaryComponent getDictionaryService(String alternativeDictionary) { @@ -1371,7 +1379,10 @@ public class SolrInformationServer implements InformationServer public void dirtyTransaction(long txnId) { this.cleanContentCache.remove(txnId); - this.cleanCascadeCache.remove(txnId); + if (cascadeTrackingEnabled()) + { + this.cleanCascadeCache.remove(txnId); + } } @Override @@ -1460,9 +1471,9 @@ public class SolrInformationServer implements InformationServer long start = System.nanoTime(); if ((node.getStatus() == SolrApiNodeStatus.DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) - || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) + || (node.getStatus() == SolrApiNodeStatus.UNKNOWN) + || cascadeTrackingEnabled() && ((node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) + || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED))) { // fix up any secondary paths NodeMetaDataParameters nmdp = new NodeMetaDataParameters(); @@ -1470,8 +1481,8 @@ public class SolrInformationServer implements InformationServer nmdp.setToNodeId(node.getId()); List nodeMetaDatas; if ((node.getStatus() == SolrApiNodeStatus.DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) + || cascadeTrackingEnabled() && ((node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) + || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED))) { // Fake the empty node metadata for this parent deleted node NodeMetaData nodeMetaData = createDeletedNodeMetaData(node); @@ -1481,7 +1492,7 @@ public class SolrInformationServer implements InformationServer { nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE); } - + NodeMetaData nodeMetaData; if (!nodeMetaDatas.isEmpty()) { @@ -1499,7 +1510,7 @@ public class SolrInformationServer implements InformationServer finally { unlock(nodeMetaData.getId()); - } + } } } // else, the node has moved on to a later transaction, and it will be indexed later @@ -1508,10 +1519,9 @@ public class SolrInformationServer implements InformationServer deleteNode(processor, request, node); } - - if ((node.getStatus() == SolrApiNodeStatus.UPDATED) - || (node.getStatus() == SolrApiNodeStatus.UNKNOWN) - || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) + if (node.getStatus() == SolrApiNodeStatus.UPDATED + || node.getStatus() == SolrApiNodeStatus.UNKNOWN + || (cascadeTrackingEnabled() && node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) { long nodeId = node.getId(); @@ -1796,8 +1806,13 @@ public class SolrInformationServer implements InformationServer EnumMap> nodeStatusToNodeIds = new EnumMap<>(SolrApiNodeStatus.class); categorizeNodes(nodes, nodeIdsToNodes, nodeStatusToNodeIds); List deletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.DELETED)); - List shardDeletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED)); - List shardUpdatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED)); + List shardDeletedNodeIds = Collections.emptyList(); + List shardUpdatedNodeIds = Collections.emptyList(); + if (cascadeTrackingEnabled()) + { + shardDeletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED)); + shardUpdatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED)); + } List unknownNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UNKNOWN)); List updatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UPDATED)); @@ -1881,7 +1896,7 @@ public class SolrInformationServer implements InformationServer continue; } - if (nodeIdsToNodes.get(nodeMetaData.getId()).getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) + if (cascadeTrackingEnabled() && nodeIdsToNodes.get(nodeMetaData.getId()).getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) { if (nodeMetaData.getProperties().get(ContentModel.PROP_CASCADE_TX) != null) { @@ -1982,7 +1997,7 @@ public class SolrInformationServer implements InformationServer { StringPropertyValue latProp = ((StringPropertyValue)nodeMetaData.getProperties().get(ContentModel.PROP_LATITUDE)); StringPropertyValue lonProp = ((StringPropertyValue)nodeMetaData.getProperties().get(ContentModel.PROP_LONGITUDE)); - + if((latProp != null) && (lonProp != null)) { String lat = latProp.getValue(); @@ -2020,14 +2035,14 @@ public class SolrInformationServer implements InformationServer } doc.addField(FIELD_ISNODE, "T"); // FIELD_FTSSTATUS is set when adding content properties to indicate whether or not the cache is clean. - + doc.addField(FIELD_TENANT, AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain())); updatePathRelatedFields(nodeMetaData, doc); updateNamePathRelatedFields(nodeMetaData, doc); updateAncestorRelatedFields(nodeMetaData, doc); doc.addField(FIELD_PARENT_ASSOC_CRC, nodeMetaData.getParentAssocsCrc()); - + if (nodeMetaData.getOwner() != null) { doc.addField(FIELD_OWNER, nodeMetaData.getOwner()); @@ -2113,7 +2128,7 @@ public class SolrInformationServer implements InformationServer } } - static void addPropertiesToDoc(Map properties, boolean isContentIndexedForNode, + static void addPropertiesToDoc(Map properties, boolean isContentIndexedForNode, SolrInputDocument newDoc, SolrInputDocument cachedDoc, boolean transformContentFlag) { for (Entry property : properties.entrySet()) @@ -2121,7 +2136,7 @@ public class SolrInformationServer implements InformationServer QName propertyQName = property.getKey(); newDoc.addField(FIELD_PROPERTIES, propertyQName.toString()); newDoc.addField(FIELD_PROPERTIES, propertyQName.getPrefixString()); - + PropertyValue value = property.getValue(); if(value != null) { @@ -2195,22 +2210,22 @@ public class SolrInformationServer implements InformationServer } } } - + private void deleteErrorNode(UpdateRequestProcessor processor, SolrQueryRequest request, Node node) throws IOException { - + String errorDocId = PREFIX_ERROR + node.getId(); - + // Try finding the node before performing removal operation DocSet docSet = request.getSearcher().getDocSet(new TermQuery(new Term(FIELD_SOLR4_ID, errorDocId))); - + if (docSet.size() > 0) { DeleteUpdateCommand delErrorDocCmd = new DeleteUpdateCommand(request); delErrorDocCmd.setId(errorDocId); processor.processDelete(delErrorDocCmd); } - + } private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, Node node) throws IOException @@ -2221,20 +2236,20 @@ public class SolrInformationServer implements InformationServer // MNT-13767 fix, remove by node DBID. deleteNode(processor, request, node.getId()); } - + private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, long dbid) throws IOException { - + // Try finding the node before performing removal operation DocSet docSet = request.getSearcher().getDocSet(LongPoint.newExactQuery(FIELD_DBID, dbid)); - + if (docSet.size() > 0) { DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request); delDocCmd.setQuery(FIELD_DBID + ":" + dbid); processor.processDelete(delDocCmd); } - + } private boolean isContentIndexedForNode(Map properties) @@ -2296,7 +2311,7 @@ public class SolrInformationServer implements InformationServer return fieldName; } - private void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, + private void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, AlfrescoSolrDataModel.ContentFieldType type, GetTextContentResponse textContentResponse) { IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getIndexedFieldForContentPropertyMetadata( @@ -2322,7 +2337,7 @@ public class SolrInformationServer implements InformationServer } } - private static void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, + private static void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName, ContentPropertyValue contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType type) { IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getIndexedFieldForContentPropertyMetadata( @@ -2352,8 +2367,8 @@ public class SolrInformationServer implements InformationServer } } } - - private static void addContentPropertyToDocUsingCache(SolrInputDocument newDoc, SolrInputDocument cachedDoc, + + private static void addContentPropertyToDocUsingCache(SolrInputDocument newDoc, SolrInputDocument cachedDoc, QName propertyQName, ContentPropertyValue contentPropertyValue, boolean transformContentFlag) { addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.DOCID); @@ -2361,14 +2376,14 @@ public class SolrInformationServer implements InformationServer addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.LOCALE); addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.MIMETYPE); addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType.ENCODING); - + if (!transformContentFlag) { // Marks it as Clean so we do not get the actual content markFTSStatus(newDoc, FTSStatus.Clean); return; } - + if (cachedDoc != null) { ofNullable(cachedDoc.getField("MINHASH")) @@ -2386,7 +2401,7 @@ public class SolrInformationServer implements InformationServer addFieldIfNotSet(newDoc, field); } - String transformationStatusFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, + String transformationStatusFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS); if (transformationStatusFieldName != null){ newDoc.addField(transformationStatusFieldName, cachedDoc.getFieldValue(transformationStatusFieldName)); @@ -2405,19 +2420,19 @@ public class SolrInformationServer implements InformationServer // Gets the new content docid and compares to that of the cachedDoc to mark the content as clean/dirty String fldName = getSolrFieldNameForContentPropertyMetadata(propertyQName, AlfrescoSolrDataModel.ContentFieldType.DOCID); - + if(newDoc.getFieldValue(FIELD_FTSSTATUS) == null) { newDoc.addField(FIELD_FTSSTATUS, cachedDoc.getFieldValue(FIELD_FTSSTATUS)); } - + if(cachedDoc.getFieldValue(fldName) != null) { long cachedDocContentDocid = Long.parseLong(String.valueOf(cachedDoc.getFieldValue(fldName))); long currentContentDocid = contentPropertyValue.getId(); // If we have used out of date content we mark it as dirty // Otherwise we leave it alone - it could already be marked as dirty/New and require an update - + if (cachedDocContentDocid != currentContentDocid) { // The cached content is out of date @@ -2429,7 +2444,7 @@ public class SolrInformationServer implements InformationServer markFTSStatus(newDoc, FTSStatus.Dirty); } } - else + else { // There is not a SolrInputDocument in the solrContentStore, so no content is added now to the new solr doc markFTSStatus(newDoc, FTSStatus.New); @@ -2468,7 +2483,7 @@ public class SolrInformationServer implements InformationServer doc.removeField(FIELD_FTSSTATUS); doc.addField(FIELD_FTSSTATUS, status.toString()); } - + private void addContentToDoc(SolrInputDocument doc, long dbId) throws AuthenticationException, IOException { Collection fieldNames = doc.deepCopy().getFieldNames(); @@ -2484,22 +2499,22 @@ public class SolrInformationServer implements InformationServer // Could update multi content but it is broken .... } } - + private void addContentPropertyToDocUsingAlfrescoRepository(SolrInputDocument doc, QName propertyQName, long dbId, String locale) throws AuthenticationException, IOException { long start = System.nanoTime(); - + // Expensive call to be done with ContentTracker GetTextContentResponse response = repositoryClient.getTextContent(dbId, propertyQName, null); - + addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS, response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_EXCEPTION, response); addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME, response); - + InputStream ris = response.getContent(); if (Objects.equals(response.getContentEncoding(), "gzip")) { @@ -2545,7 +2560,7 @@ public class SolrInformationServer implements InformationServer long end = System.nanoTime(); this.getTrackerStats().addDocTransformationTime(end - start); - + StringBuilder builder = new StringBuilder(textContent.length() + 16); builder.append("\u0000").append(locale).append("\u0000"); builder.append(textContent); @@ -2567,7 +2582,7 @@ public class SolrInformationServer implements InformationServer } private static void addMLTextPropertyToDoc(SolrInputDocument doc, FieldInstance field, MLTextPropertyValue mlTextPropertyValue) - { + { if(field.isLocalised()) { StringBuilder sort = new StringBuilder(128); @@ -2575,20 +2590,20 @@ public class SolrInformationServer implements InformationServer { final String propValue = mlTextPropertyValue.getValue(locale); LOGGER.debug("ML {} in {} of {}", field.getField(), locale, propValue); - + if((locale == null) || (propValue == null)) { continue; - } - + } + StringBuilder builder = new StringBuilder(propValue.length() + 16); builder.append("\u0000").append(locale.toString()).append("\u0000").append(propValue); - + if(!field.isSort()) { doc.addField(field.getField(), builder.toString()); } - + if (sort.length() > 0) { sort.append("\u0000"); @@ -2629,7 +2644,10 @@ public class SolrInformationServer implements InformationServer input.addField(FIELD_INTXID, txn.getId()); input.addField(FIELD_TXCOMMITTIME, txn.getCommitTimeMs()); input.addField(FIELD_DOC_TYPE, DOC_TYPE_TX); - input.addField(FIELD_CASCADE_FLAG, 0); + if (cascadeTrackingEnabled()) + { + input.addField(FIELD_CASCADE_FLAG, 0); + } cmd.solrDoc = input; processor.processAdd(cmd); } @@ -2670,8 +2688,11 @@ public class SolrInformationServer implements InformationServer input.addField(FIELD_S_TXID, info.getId()); input.addField(FIELD_S_TXCOMMITTIME, info.getCommitTimeMs()); - //Set the cascade flag to 1. This means cascading updates have not been done yet. - input.addField(FIELD_CASCADE_FLAG, 1); + if (cascadeTrackingEnabled()) + { + //Set the cascade flag to 1. This means cascading updates have not been done yet. + input.addField(FIELD_CASCADE_FLAG, 1); + } cmd.solrDoc = input; processor.processAdd(cmd); @@ -2898,7 +2919,7 @@ public class SolrInformationServer implements InformationServer { activeTrackerThreadsLock.writeLock().unlock(); } - + } @Override @@ -2925,7 +2946,7 @@ public class SolrInformationServer implements InformationServer SolrIndexSearcher solrIndexSearcher = refCounted.get(); NumericDocValues dbidDocValues = solrIndexSearcher.getSlowAtomicReader().getNumericDocValues(QueryConstants.FIELD_DBID); - + List batch = new ArrayList<>(200); DocList docList = cloud.getDocList(nativeRequestHandler, request, query.startsWith("{") ? query : "{!afts}"+query); for (DocIterator it = docList.iterator(); it.hasNext(); /**/) @@ -2940,7 +2961,7 @@ public class SolrInformationServer implements InformationServer node.setTxnId(Long.MAX_VALUE); batch.add(node); - + if(batch.size() >= 200) { indexNodes(batch, true, true); @@ -3438,7 +3459,7 @@ public class SolrInformationServer implements InformationServer SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet stack) throws AuthenticationException, IOException, JSONException { - + // skipDescendantDocsForSpecificAspects is initialised on a synchronised method, so access must be also synchronised synchronized (this) { From 6b1c8c4a1cb932d9b820b7023be2e474be4000aa Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 09:39:47 +0000 Subject: [PATCH 125/129] SEARCH-1949 Remove unused cascade parameter. --- .../org/alfresco/solr/InformationServer.java | 2 +- .../alfresco/solr/SolrInformationServer.java | 42 +++++++++++++++---- .../solr/tracker/MetadataTracker.java | 2 +- .../solr/tracker/MetadataTrackerTest.java | 2 +- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java index ec64e11b6..2e54bdb46 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java @@ -88,7 +88,7 @@ public interface InformationServer extends InformationServerCollectionProvider void indexNode(Node node, boolean overwrite) throws IOException, AuthenticationException, JSONException; - void indexNodes(List nodes, boolean overwrite, boolean cascade) throws IOException, AuthenticationException, JSONException; + void indexNodes(List nodes, boolean overwrite) throws IOException, AuthenticationException, JSONException; void cascadeNodes(List nodes, boolean overwrite) throws IOException, AuthenticationException, JSONException; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index fe94569ff..67dd1b0ea 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -19,6 +19,7 @@ package org.alfresco.solr; import static java.util.Optional.ofNullable; + import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLID; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXCOMMITTIME; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXID; @@ -76,17 +77,32 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import com.carrotsearch.hppc.IntArrayList; - import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.cursors.LongCursor; + import org.alfresco.httpclient.AuthenticationException; import org.alfresco.model.ContentModel; import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; @@ -142,7 +158,19 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.LegacyNumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.util.BytesRefBuilder; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -167,8 +195,6 @@ import org.apache.solr.search.DelegatingCollector; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.DocSet; -import org.apache.solr.search.QueryCommand; -import org.apache.solr.search.QueryResult; import org.apache.solr.search.QueryWrapperFilter; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.update.AddUpdateCommand; @@ -1795,7 +1821,7 @@ public class SolrInformationServer implements InformationServer @Override - public void indexNodes(List nodes, boolean overwrite, boolean cascade) throws IOException, JSONException + public void indexNodes(List nodes, boolean overwrite) throws IOException, JSONException { UpdateRequestProcessor processor = null; try (SolrQueryRequest request = newSolrQueryRequest()) @@ -2964,13 +2990,13 @@ public class SolrInformationServer implements InformationServer if(batch.size() >= 200) { - indexNodes(batch, true, true); + indexNodes(batch, true); batch.clear(); } } if(batch.size() > 0) { - indexNodes(batch, true, true); + indexNodes(batch, true); batch.clear(); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 4821278b4..0501108bd 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -957,7 +957,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker List filteredNodes = filterNodes(nodes); if(filteredNodes.size() > 0) { - this.infoServer.indexNodes(filteredNodes, true, false); + this.infoServer.indexNodes(filteredNodes, true); } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java index 5c68f5974..7d2c4c1ce 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/MetadataTrackerTest.java @@ -120,7 +120,7 @@ public class MetadataTrackerTest this.metadataTracker.doTrack(); InOrder inOrder = inOrder(srv); - inOrder.verify(srv).indexNodes(nodes, true, false); + inOrder.verify(srv).indexNodes(nodes, true); inOrder.verify(srv).indexTransaction(tx, true); inOrder.verify(srv).commit(); } From f23c082f1cbeecdb478e75db18282a24c9851ce9 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 13:26:38 +0000 Subject: [PATCH 126/129] SEARCH-1949 Refactor tracker registration. --- .../alfresco/solr/SolrInformationServer.java | 6 ++- .../solr/lifecycle/SolrCoreLoadListener.java | 40 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 67dd1b0ea..e5cd5f018 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -255,6 +255,8 @@ public class SolrInformationServer implements InformationServer */ private static final int BATCH_FACET_TXS = 4096; private static final String FINGERPRINT_FIELD = "MINHASH"; + /** Shared property to determine if the cascade tracking is enabled. */ + public static final String CASCADE_TRACKER_ENABLED = "alfresco.cascade.tracker.enabled"; private final AlfrescoCoreAdminHandler adminHandler; private final SolrCore core; @@ -484,9 +486,9 @@ public class SolrInformationServer implements InformationServer * * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). */ - private boolean cascadeTrackingEnabled() + public boolean cascadeTrackingEnabled() { - String cascadeTrackerEnabledProp = ofNullable((String) props.get("alfresco.cascade.tracker.enabled")).orElse("true"); + String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true"); return Boolean.valueOf(cascadeTrackerEnabledProp); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java index fc7597a0f..2a7c2b2ce 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java @@ -21,6 +21,15 @@ package org.alfresco.solr.lifecycle; import static java.util.Arrays.asList; import static java.util.Optional.ofNullable; +import static org.alfresco.solr.SolrInformationServer.CASCADE_TRACKER_ENABLED; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.function.Function; +import java.util.function.Predicate; + import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService; import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.AlfrescoSolrDataModel; @@ -54,13 +63,6 @@ import org.apache.solr.search.SolrIndexSearcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Properties; -import java.util.function.Function; -import java.util.function.Predicate; - /** * Listeners for *FIRST SEARCHER* events in order to prepare and register the SolrContentStore and the Tracking Subsystem. * @@ -257,19 +259,27 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener trackerRegistry, scheduler); - CascadeTracker cascadeTracker = - registerAndSchedule( - new CascadeTracker(props, repositoryClient, core.getName(), srv), - core, - props, - trackerRegistry, - scheduler); + List trackers = new ArrayList<>(); + + String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true"); + if (Boolean.valueOf(cascadeTrackerEnabledProp)) + { + CascadeTracker cascadeTracker = + registerAndSchedule( + new CascadeTracker(props, repositoryClient, core.getName(), srv), + core, + props, + trackerRegistry, + scheduler); + trackers.add(cascadeTracker); + } //The CommitTracker will acquire these locks in order //The ContentTracker will likely have the longest runs so put it first to ensure the MetadataTracker is not paused while //waiting for the ContentTracker to release it's lock. //The aclTracker will likely have the shortest runs so put it last. - return asList(cascadeTracker, contentTracker, metadataTracker, aclTracker); + trackers.addAll(asList(contentTracker, metadataTracker, aclTracker)); + return trackers; } /** From 944b67f0cde668b9a031778a13479289abe15cf5 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 14:28:56 +0000 Subject: [PATCH 127/129] SEARCH-1949 Update other trackers. --- .../alfresco/solr/tracker/CommitTracker.java | 22 +++++++++++++------ .../solr/tracker/MetadataTracker.java | 7 ++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index 375c21ad8..6f7332035 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -19,7 +19,11 @@ package org.alfresco.solr.tracker; +import static java.util.Optional.empty; +import static java.util.Optional.ofNullable; + import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; @@ -41,7 +45,8 @@ public class CommitTracker extends AbstractTracker private MetadataTracker metadataTracker; private AclTracker aclTracker; private ContentTracker contentTracker; - private CascadeTracker cascadeTracker; + /** The cascade tracker. Note that this may be empty if cascade tracking is disabled. */ + private Optional cascadeTracker = empty(); private AtomicInteger rollbackCount = new AtomicInteger(0); protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class); @@ -71,7 +76,7 @@ public class CommitTracker extends AbstractTracker } else if(tracker instanceof ContentTracker) { this.contentTracker = (ContentTracker)tracker; } else if(tracker instanceof CascadeTracker) { - this.cascadeTracker = (CascadeTracker)tracker; + this.cascadeTracker = ofNullable((CascadeTracker) tracker); } } @@ -182,8 +187,11 @@ public class CommitTracker extends AbstractTracker contentTracker.getWriteLock().acquire(); assert(contentTracker.getWriteLock().availablePermits() == 0); - cascadeTracker.getWriteLock().acquire(); - assert(cascadeTracker.getWriteLock().availablePermits() == 0); + if (cascadeTracker.isPresent()) + { + cascadeTracker.get().getWriteLock().acquire(); + assert (cascadeTracker.get().getWriteLock().availablePermits() == 0); + } infoSrv.rollback(); } @@ -206,12 +214,12 @@ public class CommitTracker extends AbstractTracker contentTracker.invalidateState(); //Reset cascadeTracker - cascadeTracker.setRollback(false); - cascadeTracker.invalidateState(); + cascadeTracker.ifPresent(c -> c.setRollback(false)); + cascadeTracker.ifPresent(c -> invalidateState()); //Release the locks contentTracker.getWriteLock().release(); - cascadeTracker.getWriteLock().release(); + cascadeTracker.ifPresent(c -> c.getWriteLock().release()); rollbackCount.incrementAndGet(); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 0501108bd..3df245e4a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -32,6 +32,7 @@ import org.alfresco.repo.index.shard.ShardState; import org.alfresco.solr.BoundedDeque; import org.alfresco.solr.InformationServer; import org.alfresco.solr.NodeReport; +import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.TrackerState; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.GetNodesParameters; @@ -83,6 +84,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME} */ private boolean txIntervalCommitTimeServiceAvailable = false; + /** Whether the cascade tracking is enabled. */ + private boolean cascadeTrackerEnabled = true; public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -107,6 +110,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); + cascadeTrackerEnabled = ((SolrInformationServer) informationServer).cascadeTrackingEnabled(); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 @@ -977,9 +981,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { filteredList.add(node); } - else + else if (cascadeTrackerEnabled) { - if(node.getStatus() == SolrApiNodeStatus.UPDATED) { Node doCascade = new Node(); From 12cf62abb84e9c3ad84368465ae937eca3e7e7a6 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 14:56:12 +0000 Subject: [PATCH 128/129] SEARCH-1949 Unit test for SolrCoreLoadListener. --- .../lifecycle/SolrCoreLoadListenerTest.java | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java index e1ec88140..233bfa393 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/lifecycle/SolrCoreLoadListenerTest.java @@ -18,6 +18,30 @@ */ package org.alfresco.solr.lifecycle; +import static java.util.Arrays.asList; + +import static org.alfresco.solr.SolrInformationServer.CASCADE_TRACKER_ENABLED; +import static org.alfresco.solr.tracker.Tracker.Type.ACL; +import static org.alfresco.solr.tracker.Tracker.Type.CASCADE; +import static org.alfresco.solr.tracker.Tracker.Type.CONTENT; +import static org.alfresco.solr.tracker.Tracker.Type.METADATA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.tracker.AclTracker; @@ -26,6 +50,7 @@ import org.alfresco.solr.tracker.ContentTracker; import org.alfresco.solr.tracker.MetadataTracker; import org.alfresco.solr.tracker.SolrTrackerScheduler; import org.alfresco.solr.tracker.Tracker; +import org.alfresco.solr.tracker.Tracker.Type; import org.alfresco.solr.tracker.TrackerRegistry; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; @@ -36,20 +61,6 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.xml.sax.InputSource; -import java.util.List; -import java.util.Properties; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - /** * Unit tests for the {@link SolrCoreLoadListener}. * @@ -83,6 +94,8 @@ public class SolrCoreLoadListenerTest @Before public void setUp() { + initMocks(this); + listener = new SolrCoreLoadListener(core); when(core.getName()).thenReturn(coreName); @@ -104,7 +117,29 @@ public class SolrCoreLoadListenerTest verify(scheduler).schedule(any(MetadataTracker.class), eq(coreName), same(coreProperties)); verify(scheduler).schedule(any(CascadeTracker.class), eq(coreName), same(coreProperties)); - assertEquals(4, coreTrackers.size()); + Set trackerTypes = coreTrackers.stream().map(Tracker::getType).collect(Collectors.toSet()); + assertEquals("Unexpected trackers found.", Set.of(ACL, CONTENT, METADATA, CASCADE), trackerTypes); + } + + @Test + public void testDisabledCascadeTracking() + { + coreProperties.put(CASCADE_TRACKER_ENABLED, "false"); + + List coreTrackers = listener.createAndScheduleCoreTrackers(core, registry, coreProperties, scheduler, api, informationServer); + + verify(registry).register(eq(coreName), any(AclTracker.class)); + verify(registry).register(eq(coreName), any(ContentTracker.class)); + verify(registry).register(eq(coreName), any(MetadataTracker.class)); + verify(registry, never()).register(eq(coreName), any(CascadeTracker.class)); + + verify(scheduler).schedule(any(AclTracker.class), eq(coreName), same(coreProperties)); + verify(scheduler).schedule(any(ContentTracker.class), eq(coreName), same(coreProperties)); + verify(scheduler).schedule(any(MetadataTracker.class), eq(coreName), same(coreProperties)); + verify(scheduler, never()).schedule(any(CascadeTracker.class), eq(coreName), same(coreProperties)); + + Set trackerTypes = coreTrackers.stream().map(Tracker::getType).collect(Collectors.toSet()); + assertEquals("Unexpected trackers found.", Set.of(ACL, CONTENT, METADATA), trackerTypes); } @Test From f2966a73ab29f19a9d49d55ca301486c248470f2 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 24 Jan 2020 15:54:36 +0000 Subject: [PATCH 129/129] SEARCH-1949 Add cascade enabled check to interface. --- .../src/main/java/org/alfresco/solr/InformationServer.java | 7 +++++++ .../main/java/org/alfresco/solr/SolrInformationServer.java | 7 ++----- .../java/org/alfresco/solr/tracker/MetadataTracker.java | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java index 2e54bdb46..e1b24d82b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java @@ -181,4 +181,11 @@ public interface InformationServer extends InformationServerCollectionProvider String getBaseUrl(); void flushContentStore() throws IOException; + + /** + * Check if cascade tracking is enabled. + * + * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). + */ + boolean cascadeTrackingEnabled(); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index e5cd5f018..1008789cd 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -481,11 +481,8 @@ public class SolrInformationServer implements InformationServer return this.adminHandler; } - /** - * Check if cascade tracking is enabled. - * - * @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file). - */ + /** {@inheritDoc} */ + @Override public boolean cascadeTrackingEnabled() { String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true"); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 3df245e4a..3a7773dd9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -110,7 +110,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); - cascadeTrackerEnabled = ((SolrInformationServer) informationServer).cascadeTrackingEnabled(); + cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled(); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2