From 8cf963bbbe12e1fa9d55d143316684d2377fb0d8 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 5 Sep 2019 16:44:51 +0200 Subject: [PATCH 01/23] [SEARCH-1829] Update shardProperty at each step. Improved log for shardPropertyUpdate. set shardProperty as Optional because it may miss --- .../solr/tracker/MetadataTracker.java | 50 ++++++++++++------- ...istributedDateAbstractSolrTrackerTest.java | 12 ++--- ...butedDateMonthAlfrescoSolrTrackerTest.java | 14 +++--- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index aabfb2f40..4570c0f05 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -19,12 +19,7 @@ package org.alfresco.solr.tracker; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Properties; +import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import org.alfresco.error.AlfrescoRuntimeException; @@ -84,7 +79,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker /** The string representation of the shard key. */ private String shardKey; /** The property to use for determining the shard. */ - private QName shardProperty; + private Optional shardProperty; public MetadataTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -101,13 +96,24 @@ public class MetadataTracker extends AbstractTracker implements Tracker /** * Set the shard property using the shard key. + * The property has to be update ad each iteration because the model could be deactivated or changes. */ private void updateShardProperty() { - if(shardProperty == null && shardKey != null) + Optional updatedShardProperty = getShardProperty(shardKey); + + if (shardProperty == null || !shardProperty.equals(updatedShardProperty)) { - shardProperty = getShardProperty(shardKey); + if (updatedShardProperty.isEmpty()) + { + log.warn("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKey + ", but no such property was found."); + } + else + { + log.warn("New SHARD_KEY_KEY property found for " + shardKey); + } } + shardProperty = updatedShardProperty; } MetadataTracker() @@ -174,7 +180,6 @@ public class MetadataTracker extends AbstractTracker implements Tracker * will pull its data from a "tracking" Solr node using Solr's master/slave replication, rather then tracking the repository. * */ - ShardState shardstate = getShardState(); client.getTransactions(0L, null, 0L, null, 0, shardstate); return; @@ -240,8 +245,12 @@ public class MetadataTracker extends AbstractTracker implements Tracker propertyBag.put("coreName", coreName); HashMap extendedPropertyBag = new HashMap<>(propertyBag); updateShardProperty(); - extendedPropertyBag.putAll(docRouter.getProperties(shardProperty)); - + + if (shardProperty.isPresent()) + { + extendedPropertyBag.putAll(docRouter.getProperties(shardProperty.get())); + } + return ShardStateBuilder.shardState() .withMaster(isMaster) .withLastUpdated(System.currentTimeMillis()) @@ -382,7 +391,11 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); updateShardProperty(); - gnp.setShardProperty(shardProperty); + + if (shardProperty.isPresent()) + { + gnp.setShardProperty(shardProperty.get()); + } gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, (int) info.getUpdates()); @@ -903,7 +916,10 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); updateShardProperty(); - gnp.setShardProperty(shardProperty); + if (shardProperty.isPresent()) + { + gnp.setShardProperty(shardProperty.get()); + } gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, Integer.MAX_VALUE); @@ -1203,7 +1219,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker this.queriesToReindex.offer(query); } - public static QName getShardProperty(String field) + public static Optional getShardProperty(String field) { if (StringUtils.isBlank(field)) { @@ -1218,8 +1234,8 @@ public class MetadataTracker extends AbstractTracker implements Tracker field); if (propertyDef == null) { - throw new IllegalStateException("Sharding property " + SHARD_KEY_KEY + " was set to " + field + ", but no such property was found."); + return Optional.empty(); } - return propertyDef.getName(); + return of(propertyDef.getName()); } } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java index 29cb84f21..d91a8d5c4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java @@ -20,6 +20,7 @@ package org.alfresco.solr.tracker; import org.alfresco.model.ContentModel; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; +import org.alfresco.service.namespace.QName; import org.alfresco.solr.AbstractAlfrescoDistributedTest; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.client.Acl; @@ -38,11 +39,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.junit.Test; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.List; +import java.util.*; import static java.util.Collections.singletonList; import static java.util.stream.IntStream.range; @@ -120,8 +117,11 @@ public abstract class DistributedDateAbstractSolrTrackerTest extends AbstractAlf indexTransaction(bigTxn, nodes, nodeMetaDatas); waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000); + Optional shardProperty = MetadataTracker.getShardProperty("created"); + assertTrue(shardProperty.isPresent()); + List fieldInstanceList = - AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(MetadataTracker.getShardProperty("created")).getFields(); + AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(shardProperty.get()).getFields(); AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0); String fieldName = fieldInstance.getField(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java index 4e4c03541..f11a5833c 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java @@ -32,6 +32,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; import org.alfresco.model.ContentModel; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; +import org.alfresco.service.namespace.QName; import org.alfresco.solr.AbstractAlfrescoDistributedTest; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.SolrInformationServer; @@ -52,13 +53,7 @@ import org.junit.BeforeClass; import org.junit.Test; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.List; -import java.util.Properties; -import java.util.TimeZone; +import java.util.*; @SolrTestCaseJ4.SuppressSSL public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest @@ -139,7 +134,10 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000); waitForDocCountAllCores(new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_ACL)), numAcls, 100000); - List fieldInstanceList = AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(MetadataTracker.getShardProperty("created")).getFields(); + Optional shardProperty = MetadataTracker.getShardProperty("created"); + assertTrue(shardProperty.isPresent()); + + List fieldInstanceList = AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(shardProperty.get()).getFields(); AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0); String fieldName = fieldInstance.getField(); From 21e422e11b3933be9b83a84d982bd2716af7f4ca Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 5 Sep 2019 16:46:32 +0200 Subject: [PATCH 02/23] [SEARCH-1829] set level log for shardProperty update to INFO --- .../main/java/org/alfresco/solr/tracker/MetadataTracker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 4570c0f05..443877d40 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -106,11 +106,11 @@ public class MetadataTracker extends AbstractTracker implements Tracker { if (updatedShardProperty.isEmpty()) { - log.warn("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKey + ", but no such property was found."); + log.info("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKey + ", but no such property was found."); } else { - log.warn("New SHARD_KEY_KEY property found for " + shardKey); + log.info("New SHARD_KEY_KEY property found for " + shardKey); } } shardProperty = updatedShardProperty; From 6742a4d9a9af718e85fb62fc5ba2b57152a11d07 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 6 Sep 2019 09:35:34 +0200 Subject: [PATCH 03/23] [SEARCH-1829] Improved log. Removed case in which shardProperty is null. (Empty optional instead) --- .../solr/tracker/MetadataTracker.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 443877d40..7922ae675 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -52,6 +52,8 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.swing.text.html.Option; + import static java.util.Optional.of; import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_KEY_KEY; @@ -79,7 +81,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker /** The string representation of the shard key. */ private String shardKey; /** The property to use for determining the shard. */ - private Optional shardProperty; + private Optional shardProperty = Optional.empty(); public MetadataTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) @@ -88,7 +90,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); shardKey = p.getProperty(SHARD_KEY_KEY); - updateShardProperty(); + firstUpdateShardProperty(); docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); @@ -100,22 +102,38 @@ public class MetadataTracker extends AbstractTracker implements Tracker */ private void updateShardProperty() { - Optional updatedShardProperty = getShardProperty(shardKey); - - if (shardProperty == null || !shardProperty.equals(updatedShardProperty)) + if (shardKey != null) { - if (updatedShardProperty.isEmpty()) + Optional updatedShardProperty = getShardProperty(shardKey); + if (!shardProperty.equals(updatedShardProperty)) { - log.info("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKey + ", but no such property was found."); + if (updatedShardProperty.isEmpty()) + { + log.warn("The model defining " + shardKey + " property has been disabled"); + } + else + { + log.info("New SHARD_KEY_KEY property found for " + shardKey); + } } - else + shardProperty = updatedShardProperty; + } + } + + private void firstUpdateShardProperty() + { + if (shardKey != null) + { + updateShardProperty(); + if (shardProperty.isEmpty()) { - log.info("New SHARD_KEY_KEY property found for " + shardKey); + log.warn("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKey + ", but no such property was found."); } } - shardProperty = updatedShardProperty; } + + MetadataTracker() { super(Tracker.Type.MetaData); From ff83b0b8c44dddb5a85b7f0954f786b30e218d4e Mon Sep 17 00:00:00 2001 From: Tom Page Date: Wed, 4 Sep 2019 10:27:58 +0100 Subject: [PATCH 04/23] SEARCH-1845 Update base Docker image. (cherry picked from commit cfbe98c128ecafc73a24dd0b15b6c7fbdb7fa996) --- search-services/packaging/src/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/Dockerfile b/search-services/packaging/src/docker/Dockerfile index 230214ca9..852223ecf 100644 --- a/search-services/packaging/src/docker/Dockerfile +++ b/search-services/packaging/src/docker/Dockerfile @@ -1,6 +1,6 @@ # Alfresco Search Services ${project.version} Docker Image -FROM alfresco/alfresco-base-java:11.0.1-openjdk-centos-7-3e4e9f4e5d6a +FROM alfresco/alfresco-base-java:11.0.1-openjdk-centos-7-6784d76a7b81 LABEL creator="Gethin James" maintainer="Alfresco Search Services Team" ENV DIST_DIR /opt/alfresco-search-services From 65588b7513df5a52cf0a18ff1d8d2580bc14667c Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Mon, 9 Sep 2019 09:59:17 +0100 Subject: [PATCH 05/23] Search-1847: Wait now extended until the content is indexed for faceting --- .../functional/searchServices/search/FacetFieldsSearchTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java index 7c0f4d129..50e75cc67 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java @@ -90,7 +90,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest restClient.authenticateUser(testUser).withCoreAPI().usingNode(textFile).updateNode(putBody); // Wait for the file to be indexed - waitForIndexing(htmlFile.getName(), true); + waitForContentIndexing(htmlFile.getContent(), true); } @Test From ca11f3783cf4c3c4df13f4829031c0495ccb5bce Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 10 Sep 2019 10:12:53 +0200 Subject: [PATCH 06/23] [SEARCH-1829] core review --- .../solr/tracker/MetadataTracker.java | 64 ++++++++++--------- ...istributedDateAbstractSolrTrackerTest.java | 9 ++- ...butedDateMonthAlfrescoSolrTrackerTest.java | 9 ++- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 7922ae675..58778d865 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -19,7 +19,13 @@ package org.alfresco.solr.tracker; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Properties; import java.util.concurrent.ConcurrentLinkedQueue; import org.alfresco.error.AlfrescoRuntimeException; @@ -52,10 +58,9 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.swing.text.html.Option; - import static java.util.Optional.of; +import static java.util.Optional.ofNullable; import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_KEY_KEY; /* @@ -79,7 +84,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker private ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue(); private DocRouter docRouter; /** The string representation of the shard key. */ - private String shardKey; + private Optional shardKey; /** The property to use for determining the shard. */ private Optional shardProperty = Optional.empty(); @@ -89,7 +94,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker super(p, client, coreName, informationServer, Tracker.Type.MetaData); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); - shardKey = p.getProperty(SHARD_KEY_KEY); + shardKey = ofNullable(p.getProperty(SHARD_KEY_KEY)); firstUpdateShardProperty(); docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); @@ -98,42 +103,38 @@ public class MetadataTracker extends AbstractTracker implements Tracker /** * Set the shard property using the shard key. - * The property has to be update ad each iteration because the model could be deactivated or changes. */ private void updateShardProperty() { - if (shardKey != null) - { - Optional updatedShardProperty = getShardProperty(shardKey); + shardKey.ifPresent(shardKeyName -> { + Optional updatedShardProperty = getShardProperty(shardKeyName); if (!shardProperty.equals(updatedShardProperty)) { if (updatedShardProperty.isEmpty()) { - log.warn("The model defining " + shardKey + " property has been disabled"); + log.warn("The model defining " + shardKeyName + " property has been disabled"); } else { - log.info("New SHARD_KEY_KEY property found for " + shardKey); + log.info("New " + SHARD_KEY_KEY + " property found for " + shardKeyName); } } shardProperty = updatedShardProperty; - } + }); } private void firstUpdateShardProperty() { - if (shardKey != null) - { + shardKey.ifPresent( shardKeyName -> { updateShardProperty(); if (shardProperty.isEmpty()) { - log.warn("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKey + ", but no such property was found."); + log.warn("Sharding property " + SHARD_KEY_KEY + " was set to " + shardKeyName + ", but no such property was found."); } - } + }); } - MetadataTracker() { super(Tracker.Type.MetaData); @@ -264,10 +265,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker HashMap extendedPropertyBag = new HashMap<>(propertyBag); updateShardProperty(); - if (shardProperty.isPresent()) - { - extendedPropertyBag.putAll(docRouter.getProperties(shardProperty.get())); - } + shardProperty.ifPresent(p -> extendedPropertyBag.putAll(docRouter.getProperties(p))); return ShardStateBuilder.shardState() .withMaster(isMaster) @@ -410,10 +408,8 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setStoreIdentifier(storeRef.getIdentifier()); updateShardProperty(); - if (shardProperty.isPresent()) - { - gnp.setShardProperty(shardProperty.get()); - } + shardProperty.ifPresent(p -> gnp.setShardProperty(p)); + gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, (int) info.getUpdates()); @@ -509,7 +505,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); gnp.setCoreName(coreName); - List nodes = client.getNodes(gnp, (int) info.getUpdates()); + List nodes = client.getNodes(gnp, (int) info.getUpdates()); for (Node node : nodes) { docCount++; @@ -934,10 +930,8 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); updateShardProperty(); - if (shardProperty.isPresent()) - { - gnp.setShardProperty(shardProperty.get()); - } + shardProperty.ifPresent(p -> gnp.setShardProperty(p)); + gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, Integer.MAX_VALUE); @@ -1237,6 +1231,16 @@ public class MetadataTracker extends AbstractTracker implements Tracker this.queriesToReindex.offer(query); } + + /** + * Given the field name, returns the name of the property definition. + * If the property definition is not found, Empty optional is returned. + * + * @param field + * + * @return the name of the associated property definition if present, Optional.Empty() otherwise + * + */ public static Optional getShardProperty(String field) { if (StringUtils.isBlank(field)) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java index d91a8d5c4..6aacd0282 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateAbstractSolrTrackerTest.java @@ -39,7 +39,12 @@ import org.apache.solr.client.solrj.SolrQuery; import org.junit.Test; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.Optional; import static java.util.Collections.singletonList; import static java.util.stream.IntStream.range; @@ -118,7 +123,7 @@ public abstract class DistributedDateAbstractSolrTrackerTest extends AbstractAlf waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000); Optional shardProperty = MetadataTracker.getShardProperty("created"); - assertTrue(shardProperty.isPresent()); + assertTrue("'created' field is expected to be found in data model", shardProperty.isPresent()); List fieldInstanceList = AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(shardProperty.get()).getFields(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java index f11a5833c..e0a569b4a 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java @@ -53,7 +53,14 @@ import org.junit.BeforeClass; import org.junit.Test; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.TimeZone; @SolrTestCaseJ4.SuppressSSL public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest From 9f4fc89a8c8aa7aa5db89d5628a0ecf06fa2b165 Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Tue, 10 Sep 2019 09:59:07 +0100 Subject: [PATCH 07/23] Search:1835: Updated RM artifacts' names and versions --- e2e-test/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index b0d9d7147..8bc984e89 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -14,7 +14,7 @@ 6.0.1.2 6.0.0.4 3.0.11 - 2.6.0 + 3.1.0 src/test/resources/SearchSuite.xml @@ -77,7 +77,7 @@ org.alfresco - alfresco-rm-automation-enterprise-rest-api + alfresco-governance-services-automation-enterprise-rest-api ${rm.version} test @@ -88,7 +88,7 @@ org.alfresco - alfresco-rm-automation-community-rest-api + alfresco-governance-services-automation-community-rest-api ${rm.version} tests test From 925c590e95a73b93d24974b546d4791f47822474 Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Tue, 10 Sep 2019 10:36:17 +0100 Subject: [PATCH 08/23] Search:1835: Allow share image location to be specified, to enable running GS share --- search-services/packaging/src/docker/6.x/.env | 1 + search-services/packaging/src/docker/6.x/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/search-services/packaging/src/docker/6.x/.env b/search-services/packaging/src/docker/6.x/.env index 37cd93f01..e4e98f347 100644 --- a/search-services/packaging/src/docker/6.x/.env +++ b/search-services/packaging/src/docker/6.x/.env @@ -1,6 +1,7 @@ # In order to start the community edition set the value of ALFRESCO_IMAGE to alfresco/alfresco-content-repository-community ALFRESCO_IMAGE=alfresco/alfresco-content-repository ALFRESCO_TAG=6.1.0-EA3 +SHARE_IMAGE=alfresco/alfresco-share SHARE_TAG=6.0 POSTGRES_TAG=10.1 SEARCH_TAG=latest \ No newline at end of file diff --git a/search-services/packaging/src/docker/6.x/docker-compose.yml b/search-services/packaging/src/docker/6.x/docker-compose.yml index 5b07ec59b..b7e4c3bbc 100644 --- a/search-services/packaging/src/docker/6.x/docker-compose.yml +++ b/search-services/packaging/src/docker/6.x/docker-compose.yml @@ -20,7 +20,7 @@ services: - "5005:5005" #Java debugging - "8081:8080" #Browser port for Alfresco share: - image: alfresco/alfresco-share:${SHARE_TAG} + image: ${SHARE_IMAGE}:${SHARE_TAG} environment: - REPO_HOST=alfresco - REPO_PORT=8080 From c970a773440c865168daba91dc95b3bd9bea0996 Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Tue, 10 Sep 2019 15:18:20 +0100 Subject: [PATCH 09/23] Search:1835: Amended the image location to be from quay.io explicitely, for consistency --- search-services/packaging/src/docker/6.x/.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/packaging/src/docker/6.x/.env b/search-services/packaging/src/docker/6.x/.env index e4e98f347..f23d97f96 100644 --- a/search-services/packaging/src/docker/6.x/.env +++ b/search-services/packaging/src/docker/6.x/.env @@ -1,7 +1,7 @@ # In order to start the community edition set the value of ALFRESCO_IMAGE to alfresco/alfresco-content-repository-community -ALFRESCO_IMAGE=alfresco/alfresco-content-repository +ALFRESCO_IMAGE=quay.io/alfresco/alfresco-content-repository ALFRESCO_TAG=6.1.0-EA3 -SHARE_IMAGE=alfresco/alfresco-share +SHARE_IMAGE=quay.io/alfresco/alfresco-share SHARE_TAG=6.0 POSTGRES_TAG=10.1 SEARCH_TAG=latest \ No newline at end of file From bb596d322558f16cc32a7b9615742029dffb266c Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Tue, 10 Sep 2019 16:05:14 +0100 Subject: [PATCH 10/23] SearchSearch:1835: Added comment for using GS Version 3.1.0 --- e2e-test/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 8bc984e89..41bf7fcd5 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -14,6 +14,8 @@ 6.0.1.2 6.0.0.4 3.0.11 + 3.1.0 src/test/resources/SearchSuite.xml From bc4a012fbd0d9f8b584435332bf741bc2bf96ba8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2019 00:24:10 +0000 Subject: [PATCH 11/23] Bump lombok from 1.18.8 to 1.18.10 in /e2e-test Bumps [lombok](https://github.com/rzwitserloot/lombok) from 1.18.8 to 1.18.10. - [Release notes](https://github.com/rzwitserloot/lombok/releases) - [Changelog](https://github.com/rzwitserloot/lombok/blob/master/doc/changelog.markdown) - [Commits](https://github.com/rzwitserloot/lombok/compare/v1.18.8...v1.18.10) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 41bf7fcd5..39f94f499 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -120,7 +120,7 @@ org.projectlombok lombok - 1.18.8 + 1.18.10 test From 17ad0455f01a747f089efee16ff3047b41279eef Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 12 Sep 2019 08:57:45 +0100 Subject: [PATCH 12/23] Use the maven compiler plugin to tell IDEs what version of Java to use. --- pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pom.xml b/pom.xml index c305c9aeb..de8e449dd 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,7 @@ 1.1.0 + 11 6.6.5 ${solr.base.version}-patched @@ -39,6 +40,20 @@ search-services insight-engine + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + ${java.version} + true + true + + + + From 0f0f7236a8b801ab0d0ecf1512a613b0d0df25b1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2019 00:14:18 +0000 Subject: [PATCH 13/23] Bump alfresco-data-model from 8.48 to 8.49 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.48 to 8.49. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.48...8.49) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 152d25d5e..6223ec5d7 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.48 + 8.49 2.10.0.pr2 From 39421555afbd088ae759d6daf60f3d5f5441f8a4 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 13 Sep 2019 10:09:05 +0200 Subject: [PATCH 14/23] [SEARCH-1829] Adding test for checking that indexing still works after sharding model has been disabled. Modified cascading sharding test in order to clean up the environment after test execution. --- .../model/sharding-content-model.xml | 27 ++++++ .../search/ExplicitRouting.java | 79 ++++++++++++++++ .../CascadingTrackerIntegrationTest.java | 92 ++++++++++++++----- 3 files changed, 175 insertions(+), 23 deletions(-) create mode 100644 e2e-test/src/main/resources/model/sharding-content-model.xml create mode 100644 e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java diff --git a/e2e-test/src/main/resources/model/sharding-content-model.xml b/e2e-test/src/main/resources/model/sharding-content-model.xml new file mode 100644 index 000000000..159d02f12 --- /dev/null +++ b/e2e-test/src/main/resources/model/sharding-content-model.xml @@ -0,0 +1,27 @@ + + + + Explicit Routing for Sharding Sample Model + + + + + + + + + + + + Sharding + + + + Shard Id + d:text + + + + + + diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java new file mode 100644 index 000000000..e0691ca5f --- /dev/null +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java @@ -0,0 +1,79 @@ +package org.alfresco.test.search.functional.searchServices.search; + +import org.alfresco.search.TestGroup; +import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; +import org.alfresco.utility.model.FileModel; +import org.alfresco.utility.model.FileType; +import org.apache.chemistry.opencmis.commons.PropertyIds; +import org.apache.chemistry.opencmis.commons.enums.VersioningState; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static org.testng.Assert.assertTrue; + +public class ExplicitRouting extends AbstractE2EFunctionalTest { + + + /** + * Checks indexing still works after sharding model used for explicit routing has been disabled + * @throws Exception + */ + @Test(priority = 1, groups = {TestGroup.NOT_BAMBOO, TestGroup.EXPLICIT_SHARDING }) + public void testIndexingStillWorkingAfterShardModelIsDeactivated() throws Exception + { + + // Deploy sharding model + assertTrue(deployCustomModel("model/sharding-content-model.xml"), + "failing while deploying sharding model"); + + // Create a first child in parent folder. It will be indexed in the parent shard (shard 0) + FileModel file = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + Map propertiesFirstChild = Map.of(PropertyIds.NAME, file.getName(), + PropertyIds.OBJECT_TYPE_ID, "cmis:document", + "cmis:secondaryObjectTypeIds", List.of("P:shard:sharding"), + "shard:shardId", "0"); + + // Create file using shard:shardId + cmisApi.authenticateUser(testUser).usingSite(testSite) + .createFile(file, + Map.of(PropertyIds.NAME, file.getName(), + PropertyIds.OBJECT_TYPE_ID, "cmis:document"), + VersioningState.MAJOR) + .assertThat().existsInRepo(); + + // Wait for file to be indexed + assertTrue(waitForMetadataIndexing(file.getName(), true), + "A file using sharding model has not been indexed"); + + + // Deleting file + dataContent.usingSite(testSite).usingUser(testUser).usingResource(file).deleteContent(); + restClient.withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(file); + + // Deleting sharding model + assertTrue(deactivateCustomModel("sharding-content-model.xml"), + "failing while deactivating sharding model"); + assertTrue(deleteCustomModel("sharding-content-model.xml"), + "failing while removing sharding model"); + + assertTrue(waitForIndexing("TYPE:'" + "shard:shardId" + "'", false), + "Indexes are not updated after deactivating a model"); + + // Create a file in the parent folder + file = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + + cmisApi.authenticateUser(testUser).usingSite(testSite) + .createFile(file, + Map.of(PropertyIds.NAME, file.getName(), + PropertyIds.OBJECT_TYPE_ID, "cmis:document"), + VersioningState.MAJOR) + .assertThat().existsInRepo(); + + + assertTrue(waitForMetadataIndexing(file.getName(), true), + "Indexing is not working after the sharding model has been removed"); + + } +} diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java index daf9e0ce0..977e12cd8 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java @@ -19,8 +19,12 @@ import org.apache.chemistry.opencmis.commons.PropertyIds; import org.apache.chemistry.opencmis.commons.enums.VersioningState; import org.springframework.beans.factory.annotation.Autowired; import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static org.testng.Assert.assertTrue; + /** * Test class tests cascading updates for a child node when parent node is updated * @@ -31,6 +35,46 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest { @Autowired protected DataContent dataContent; + private FolderModel parentFolderSharded; + private FileModel firstChildFileSharded; + private FileModel secondChildFileSharded; + + + @BeforeClass(alwaysRun = true) + public void setupEnvironment() + { + assertTrue(deployCustomModel("model/sharding-content-model.xml"), + "failing while deploying sharding model"); + } + + + @AfterClass + public void cleanUpEnvironment() + { + if (firstChildFileSharded != null) + { + dataContent.usingSite(testSite).usingUser(testUser).usingResource(firstChildFileSharded).deleteContent(); + restClient.withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(firstChildFileSharded); + } + + if (secondChildFileSharded != null) + { + dataContent.usingSite(testSite).usingUser(testUser).usingResource(secondChildFileSharded).deleteContent(); + restClient.withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(secondChildFileSharded); + } + + if (parentFolderSharded != null) + { + dataContent.usingSite(testSite).usingUser(testUser).usingResource(parentFolderSharded).deleteContent(); + restClient.withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(parentFolderSharded); + } + + dataContent.deleteSite(testSite); + assertTrue(deactivateCustomModel("sharding-content-model.xml"), + "failing while deactivating sharding model"); + assertTrue(deleteCustomModel("sharding-content-model.xml"), + "failing while removing sharding model"); + } @Test(priority = 1) public void testChildPathWhenParentRenamed() throws Exception @@ -41,6 +85,7 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest // Create a file in the parent folder FileModel childFile = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + cmisApi.authenticateUser(testUser).usingSite(testSite).usingResource(parentFolder) .createFile(childFile, Map.of(PropertyIds.NAME, childFile.getName(), @@ -79,10 +124,10 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest public void testGrandChildPathWhenGrandParentRenamed() throws Exception { // Create grand parent folder - FolderModel grandParentFolder = dataContent.usingSite(testSite).usingUser(testUser).createFolder(); + FolderModel grandParentFolderSharding = dataContent.usingSite(testSite).usingUser(testUser).createFolder(); // Create child folder - FolderModel childFolder = dataContent.usingUser(testUser).usingResource(grandParentFolder).createFolder(); + FolderModel childFolder = dataContent.usingUser(testUser).usingResource(grandParentFolderSharding).createFolder(); // Create grand child file FileModel grandChildFile = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); @@ -100,15 +145,15 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest // Query to find nodes where Path with original folder name matches String parentQuery = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + - "/cm:documentLibrary/cm:" + grandParentFolder.getName() + "/*\""; + "/cm:documentLibrary/cm:" + grandParentFolderSharding.getName() + "/*\""; // Rename grand parent folder String grandParentNewName = "grandParentRenamed"; - grandParentFolder.setName(grandParentNewName); + grandParentFolderSharding.setName(grandParentNewName); ContentModel grandParentFolderRenamed = new ContentModel(grandParentNewName); - dataContent.usingUser(testUser).usingResource(grandParentFolder).renameContent(grandParentFolderRenamed); + dataContent.usingUser(testUser).usingResource(grandParentFolderSharding).renameContent(grandParentFolderRenamed); // Find nodes where Path with new folder name matches String childrenQueryAfterRename = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + @@ -148,48 +193,48 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest { // Create Parent folder. It will be indexed in shard 0 - FolderModel parentFolder = FolderModel.getRandomFolderModel(); + parentFolderSharded = FolderModel.getRandomFolderModel(); List secondaryTypes = List.of("P:shard:sharding"); - Map parentProperties = Map.of(PropertyIds.NAME, parentFolder.getName(), + Map parentProperties = Map.of(PropertyIds.NAME, parentFolderSharded.getName(), PropertyIds.OBJECT_TYPE_ID, "cmis:folder", "cmis:secondaryObjectTypeIds", secondaryTypes, "shard:shardId", "0"); // Create a first child in parent folder. It will be indexed in the parent shard (shard 0) - FileModel firstChildFile = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); - Map propertiesFirstChild = Map.of(PropertyIds.NAME, firstChildFile.getName(), + firstChildFileSharded = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + Map propertiesFirstChild = Map.of(PropertyIds.NAME, firstChildFileSharded.getName(), PropertyIds.OBJECT_TYPE_ID, "cmis:document", "cmis:secondaryObjectTypeIds", secondaryTypes, "shard:shardId", "0"); // Create a second child in parent folder. It will be indexed in shard 1. - FileModel secondChildFile = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); - Map propertiesSecondChild = Map.of(PropertyIds.NAME, secondChildFile.getName(), + secondChildFileSharded = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + Map propertiesSecondChild = Map.of(PropertyIds.NAME, secondChildFileSharded.getName(), PropertyIds.OBJECT_TYPE_ID, "cmis:document", "cmis:secondaryObjectTypeIds", secondaryTypes, "shard:shardId", "1"); - cmisApi.authenticateUser(testUser).usingSite(testSite).createFolder(parentFolder, parentProperties).then() - .usingResource(parentFolder) - .createFile(firstChildFile, propertiesFirstChild, VersioningState.MAJOR) - .createFile(secondChildFile, propertiesSecondChild, VersioningState.MAJOR); + cmisApi.authenticateUser(testUser).usingSite(testSite).createFolder(parentFolderSharded, parentProperties).then() + .usingResource(parentFolderSharded) + .createFile(firstChildFileSharded, propertiesFirstChild, VersioningState.MAJOR) + .createFile(secondChildFileSharded, propertiesSecondChild, VersioningState.MAJOR); // Check everything is indexed - Assert.assertTrue(waitForIndexing(firstChildFile.getName(), true), "file: " + firstChildFile.getName() + " has not been indexed."); - Assert.assertTrue(waitForIndexing(secondChildFile.getName(), true), "file: " + secondChildFile.getName() + " has not been indexed."); - Assert.assertTrue(waitForIndexing(parentFolder.getName(), true), "file: " + parentFolder.getName() + " has not been indexed."); + assertTrue(waitForIndexing(firstChildFileSharded.getName(), true), "file: " + firstChildFileSharded.getName() + " has not been indexed."); + assertTrue(waitForIndexing(secondChildFileSharded.getName(), true), "file: " + secondChildFileSharded.getName() + " has not been indexed."); + assertTrue(waitForIndexing(parentFolderSharded.getName(), true), "file: " + parentFolderSharded.getName() + " has not been indexed."); // Query to find nodes where Path with original folder name matches String parentQuery = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + - "/cm:documentLibrary/cm:" + parentFolder.getName() + "/*\""; + "/cm:documentLibrary/cm:" + parentFolderSharded.getName() + "/*\""; // Rename parent folder String parentNewName = "parentRenamedSharding"; - parentFolder.setName(parentNewName); + parentFolderSharded.setName(parentNewName); ContentModel parentNewNameModel = new ContentModel(parentNewName); - dataContent.usingUser(testUser).usingResource(parentFolder).renameContent(parentNewNameModel); + dataContent.usingUser(testUser).usingResource(parentFolderSharded).renameContent(parentNewNameModel); String parentQueryAfterRename = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + "/cm:documentLibrary/cm:" + parentNewName + "/*\""; @@ -199,9 +244,9 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest Assert.assertEquals(descendantCountOfNewNameBeforeUpdate, 0, "There should be 0 results performing the new query before updating parent name"); - Assert.assertTrue(waitForMetadataIndexing(parentNewName, true), "failing while renaming " + parentFolder.getName() + " to " + parentNewName); + assertTrue(waitForMetadataIndexing(parentNewName, true), "failing while renaming " + parentFolderSharded.getName() + " to " + parentNewName); - boolean indexingInProgress = !isContentInSearchResults(parentQueryAfterRename, firstChildFile.getName(), true); + boolean indexingInProgress = !isContentInSearchResults(parentQueryAfterRename, firstChildFileSharded.getName(), true); // Query using new parent name: Expect the two children int descendantCountOfNewName = query(parentQueryAfterRename).getPagination().getCount(); @@ -210,6 +255,7 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest // Query using old parent name: Expect no descendant after rename int descendantCountOfOriginalName = query(parentQuery).getPagination().getCount(); Assert.assertEquals(descendantCountOfOriginalName, 0, "Old path still has descendants: " + parentQuery); + } } From a0f4aa4a804c5601802d6c32259a6bc0ca39f178 Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Fri, 13 Sep 2019 13:31:06 +0100 Subject: [PATCH 15/23] Search-1856: Fixed the failing APTH test --- .../searchServices/search/SearchAPATHTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java index 33592222e..9bd2050fc 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java @@ -29,7 +29,6 @@ import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.RestResultBucketsModel; import org.alfresco.rest.search.SearchRequest; import org.alfresco.rest.search.SearchResponse; -import org.alfresco.search.TestGroup; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -105,14 +104,18 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest buckets.forEach(bucket -> bucket.assertThat().field("label").contains("0/")); } + /** + * Test to test the sub-level 1/ + * Test to search for a searchString, that's unique to the test run and hence stable for any environment + */ @Test public void searchLevel0andIncludeSubLevel1() { - SearchRequest searchQuery = searchRequestWithAPATHFacet("name:*", "1/"); + SearchRequest searchQuery = searchRequestWithAPATHFacet("name:" + unique_searchString, "1/"); SearchResponse response = query(searchQuery); List buckets = getBuckets(response); - Assert.assertEquals(4, buckets.size()); + Assert.assertEquals(buckets.size(), 1, "Incorrect bucket count"); getFirstBucket(response).assertThat().field("label").contains("1/"); } From 58d98a198424a8d0b0af88616d763ae0af1f1a36 Mon Sep 17 00:00:00 2001 From: "meenal.bhave@alfresco.com" Date: Fri, 13 Sep 2019 13:31:30 +0100 Subject: [PATCH 16/23] Search-1856: Fixed the failing APTH test --- .../functional/searchServices/search/SearchAPATHTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java index 9bd2050fc..1016869cc 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java @@ -105,7 +105,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest } /** - * Test to test the sub-level 1/ + * Test to test that the facet buckets are returned correctly for sub-level 1/ * Test to search for a searchString, that's unique to the test run and hence stable for any environment */ @Test From d7daf09904bc215552c7ad6c28f4e980d633e489 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 16 Sep 2019 09:08:21 +0100 Subject: [PATCH 17/23] Make e2e test a submodule of the main project. This cannot be done on some older branches as the java versions are different, but on master everything is Java 11. Also fix target version to be Java 11, so that IDEs pick the right version of Java when running tests. --- e2e-test/pom.xml | 26 ++++++-------------------- pom.xml | 4 +++- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 39f94f499..3baa0d359 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -3,8 +3,8 @@ 4.0.0 org.alfresco - alfresco-super-pom - 10 + alfresco-search-and-insight-parent + 1.5.0-SNAPSHOT search-analytics-e2e-test search-analytics-e2e-test @@ -14,30 +14,16 @@ 6.0.1.2 6.0.0.4 3.0.11 - 3.1.0 src/test/resources/SearchSuite.xml - 11 - ${java.version} - ${java.version} - 3.8.1 2.7.7 - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${java.version} - true - true - - org.apache.maven.plugins maven-surefire-plugin @@ -65,8 +51,8 @@ - - org.alfresco.tas + + org.alfresco.tas cmis-test ${tas.cmis.api.version} test @@ -117,7 +103,7 @@ - + org.projectlombok lombok 1.18.10 diff --git a/pom.xml b/pom.xml index de8e449dd..5731b13fe 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ scm:git:https://git.alfresco.com/search_discovery/insightengine.git scm:git:https://git.alfresco.com/search_discovery/insightengine.git https://git.alfresco.com/search_discovery/insightengine.git - 1.1.0 + HEAD 11 @@ -39,6 +39,7 @@ search-services insight-engine + e2e-test @@ -48,6 +49,7 @@ 3.8.1 ${java.version} + ${java.version} true true From 9c8c0e95ca1df11fcd8418b0fc01e978cd632c0b Mon Sep 17 00:00:00 2001 From: Tom Page Date: Mon, 16 Sep 2019 09:19:50 +0100 Subject: [PATCH 18/23] Also add insight-jdbc as an explicit dependency of the e2e tests. This overrides the old version declared in tas-rest. --- e2e-test/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 3baa0d359..8d3ea4ab7 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -39,6 +39,11 @@ + + org.alfresco + alfresco-insight-jdbc + 1.5.0-SNAPSHOT + org.alfresco.tas restapi-test From ca3010e473ea86f2fe51d18a82108a52a8cd535c Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 16 Sep 2019 13:34:04 +0200 Subject: [PATCH 19/23] [SEARCH-1829] Restore tests after unwanted changes --- .../tracker/CascadingTrackerIntegrationTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java index 977e12cd8..e591c9e42 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java @@ -85,7 +85,6 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest // Create a file in the parent folder FileModel childFile = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); - cmisApi.authenticateUser(testUser).usingSite(testSite).usingResource(parentFolder) .createFile(childFile, Map.of(PropertyIds.NAME, childFile.getName(), @@ -124,10 +123,10 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest public void testGrandChildPathWhenGrandParentRenamed() throws Exception { // Create grand parent folder - FolderModel grandParentFolderSharding = dataContent.usingSite(testSite).usingUser(testUser).createFolder(); + FolderModel grandParentFolder = dataContent.usingSite(testSite).usingUser(testUser).createFolder(); // Create child folder - FolderModel childFolder = dataContent.usingUser(testUser).usingResource(grandParentFolderSharding).createFolder(); + FolderModel childFolder = dataContent.usingUser(testUser).usingResource(grandParentFolder).createFolder(); // Create grand child file FileModel grandChildFile = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); @@ -145,15 +144,15 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest // Query to find nodes where Path with original folder name matches String parentQuery = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + - "/cm:documentLibrary/cm:" + grandParentFolderSharding.getName() + "/*\""; + "/cm:documentLibrary/cm:" + grandParentFolder.getName() + "/*\""; // Rename grand parent folder String grandParentNewName = "grandParentRenamed"; - grandParentFolderSharding.setName(grandParentNewName); + grandParentFolder.setName(grandParentNewName); ContentModel grandParentFolderRenamed = new ContentModel(grandParentNewName); - dataContent.usingUser(testUser).usingResource(grandParentFolderSharding).renameContent(grandParentFolderRenamed); + dataContent.usingUser(testUser).usingResource(grandParentFolder).renameContent(grandParentFolderRenamed); // Find nodes where Path with new folder name matches String childrenQueryAfterRename = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + From fe429394f47032781c785d0c9252a684f41f2845 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 17 Sep 2019 09:36:59 +0100 Subject: [PATCH 20/23] Use project.version for insight-jdbc dependency. --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 8d3ea4ab7..10c34d1c8 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -42,7 +42,7 @@ org.alfresco alfresco-insight-jdbc - 1.5.0-SNAPSHOT + ${project.version} org.alfresco.tas From 00148a26dcae6111664216acf0d5bbce81ea979d Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 17 Sep 2019 09:41:44 +0100 Subject: [PATCH 21/23] Fix whitespace in e2e pom. --- e2e-test/pom.xml | 98 ++++++++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 10c34d1c8..926c04a09 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -3,8 +3,8 @@ 4.0.0 org.alfresco - alfresco-search-and-insight-parent - 1.5.0-SNAPSHOT + alfresco-search-and-insight-parent + 1.5.0-SNAPSHOT search-analytics-e2e-test search-analytics-e2e-test @@ -49,24 +49,24 @@ restapi-test ${tas.rest.api.version} test - - - com.fasterxml.jackson.core - jackson-databind - - + + + com.fasterxml.jackson.core + jackson-databind + + - - org.alfresco.tas + + org.alfresco.tas cmis-test ${tas.cmis.api.version} test - - - com.fasterxml.jackson.core - jackson-databind - - + + + com.fasterxml.jackson.core + jackson-databind + + org.alfresco @@ -74,50 +74,50 @@ ${rm.version} test - - com.fasterxml.jackson.core - jackson-databind - ${jackson.databind.version} - + + com.fasterxml.jackson.core + jackson-databind + ${jackson.databind.version} + org.alfresco alfresco-governance-services-automation-community-rest-api ${rm.version} tests test - - - com.fasterxml.jackson.core - jackson-databind - - + + + com.fasterxml.jackson.core + jackson-databind + + - - - org.alfresco.tas - utility - ${tas.utility.version} - - - slf4j-api - org.slf4j - - - junit - junit - - - - + + + org.alfresco.tas + utility + ${tas.utility.version} + + + slf4j-api + org.slf4j + + + junit + junit + + + + org.projectlombok lombok 1.18.10 test - org.openjfx - javafx-fxml - 12-ea+10 - + org.openjfx + javafx-fxml + 12-ea+10 + - \ No newline at end of file + From 5723834e250fb8079a2dd62f88a350e99f128440 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 17 Sep 2019 14:35:24 +0200 Subject: [PATCH 22/23] [SEARCH-1829] review some changes --- .../search/ExplicitRoutingTest.java | 79 +++++++++++++++++++ .../CascadingTrackerIntegrationTest.java | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRoutingTest.java diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRoutingTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRoutingTest.java new file mode 100644 index 000000000..541e1d1d1 --- /dev/null +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRoutingTest.java @@ -0,0 +1,79 @@ +package org.alfresco.test.search.functional.searchServices.search; + +import org.alfresco.search.TestGroup; +import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; +import org.alfresco.utility.model.FileModel; +import org.alfresco.utility.model.FileType; +import org.apache.chemistry.opencmis.commons.PropertyIds; +import org.apache.chemistry.opencmis.commons.enums.VersioningState; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static org.testng.Assert.assertTrue; + +public class ExplicitRoutingTest extends AbstractE2EFunctionalTest { + + + /** + * Checks indexing still works after sharding model used for explicit routing has been disabled + * @throws Exception + */ + @Test(priority = 1, groups = {TestGroup.NOT_BAMBOO, TestGroup.EXPLICIT_SHARDING }) + public void testIndexingStillWorkingAfterShardModelIsDeactivated() throws Exception + { + + // Deploy sharding model + assertTrue(deployCustomModel("model/sharding-content-model.xml"), + "failing while deploying sharding model"); + + // Create a first child in parent folder. It will be indexed in the parent shard (shard 0) + FileModel file = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + Map propertiesFirstChild = Map.of(PropertyIds.NAME, file.getName(), + PropertyIds.OBJECT_TYPE_ID, "cmis:document", + "cmis:secondaryObjectTypeIds", List.of("P:shard:sharding"), + "shard:shardId", "0"); + + // Create file using shard:shardId + cmisApi.authenticateUser(testUser).usingSite(testSite) + .createFile(file, + Map.of(PropertyIds.NAME, file.getName(), + PropertyIds.OBJECT_TYPE_ID, "cmis:document"), + VersioningState.MAJOR) + .assertThat().existsInRepo(); + + // Wait for file to be indexed + assertTrue(isContentInSearchResults(file.getName(), file.getName(), true), + "A file using sharding model has not been indexed"); + + + // Deleting file + dataContent.usingSite(testSite).usingUser(testUser).usingResource(file).deleteContent(); + restClient.withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(file); + + // Deleting sharding model + assertTrue(deactivateCustomModel("sharding-content-model.xml"), + "failing while deactivating sharding model"); + assertTrue(deleteCustomModel("sharding-content-model.xml"), + "failing while removing sharding model"); + + assertTrue(waitForIndexing("TYPE:'" + "shard:shardId" + "'", false), + "Indexes are not updated after deactivating a model"); + + // Create a file in the parent folder + file = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); + + cmisApi.authenticateUser(testUser).usingSite(testSite) + .createFile(file, + Map.of(PropertyIds.NAME, file.getName(), + PropertyIds.OBJECT_TYPE_ID, "cmis:document"), + VersioningState.MAJOR) + .assertThat().existsInRepo(); + + + assertTrue(isContentInSearchResults(file.getName(), file.getName(), true), + "Indexing is not working after the sharding model has been removed"); + + } +} diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java index e591c9e42..3d66f1922 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java @@ -187,7 +187,7 @@ public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest * Check that, after parent renaming, both the children are searchable in the new path * (computed accordingly with the new parent folder name) */ - @Test(priority = 1, groups = {TestGroup.NOT_BAMBOO, TestGroup.EXPLICIT_SHARDING }) + @Test(priority = 3, groups = {TestGroup.NOT_BAMBOO, TestGroup.EXPLICIT_SHARDING }) public void testChildrenPathOnParentRenamedWithChildrenInDifferentShards() throws Exception { From b6af0f2d2afde0939e6bd22f0c5f8e845e0782d2 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 17 Sep 2019 14:35:51 +0200 Subject: [PATCH 23/23] [SEARCH-1829] removed file --- .../search/ExplicitRouting.java | 79 ------------------- 1 file changed, 79 deletions(-) delete mode 100644 e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java deleted file mode 100644 index e0691ca5f..000000000 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ExplicitRouting.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.alfresco.test.search.functional.searchServices.search; - -import org.alfresco.search.TestGroup; -import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; -import org.alfresco.utility.model.FileModel; -import org.alfresco.utility.model.FileType; -import org.apache.chemistry.opencmis.commons.PropertyIds; -import org.apache.chemistry.opencmis.commons.enums.VersioningState; -import org.testng.annotations.Test; - -import java.util.List; -import java.util.Map; - -import static org.testng.Assert.assertTrue; - -public class ExplicitRouting extends AbstractE2EFunctionalTest { - - - /** - * Checks indexing still works after sharding model used for explicit routing has been disabled - * @throws Exception - */ - @Test(priority = 1, groups = {TestGroup.NOT_BAMBOO, TestGroup.EXPLICIT_SHARDING }) - public void testIndexingStillWorkingAfterShardModelIsDeactivated() throws Exception - { - - // Deploy sharding model - assertTrue(deployCustomModel("model/sharding-content-model.xml"), - "failing while deploying sharding model"); - - // Create a first child in parent folder. It will be indexed in the parent shard (shard 0) - FileModel file = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); - Map propertiesFirstChild = Map.of(PropertyIds.NAME, file.getName(), - PropertyIds.OBJECT_TYPE_ID, "cmis:document", - "cmis:secondaryObjectTypeIds", List.of("P:shard:sharding"), - "shard:shardId", "0"); - - // Create file using shard:shardId - cmisApi.authenticateUser(testUser).usingSite(testSite) - .createFile(file, - Map.of(PropertyIds.NAME, file.getName(), - PropertyIds.OBJECT_TYPE_ID, "cmis:document"), - VersioningState.MAJOR) - .assertThat().existsInRepo(); - - // Wait for file to be indexed - assertTrue(waitForMetadataIndexing(file.getName(), true), - "A file using sharding model has not been indexed"); - - - // Deleting file - dataContent.usingSite(testSite).usingUser(testUser).usingResource(file).deleteContent(); - restClient.withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(file); - - // Deleting sharding model - assertTrue(deactivateCustomModel("sharding-content-model.xml"), - "failing while deactivating sharding model"); - assertTrue(deleteCustomModel("sharding-content-model.xml"), - "failing while removing sharding model"); - - assertTrue(waitForIndexing("TYPE:'" + "shard:shardId" + "'", false), - "Indexes are not updated after deactivating a model"); - - // Create a file in the parent folder - file = FileModel.getRandomFileModel(FileType.TEXT_PLAIN, "custom content"); - - cmisApi.authenticateUser(testUser).usingSite(testSite) - .createFile(file, - Map.of(PropertyIds.NAME, file.getName(), - PropertyIds.OBJECT_TYPE_ID, "cmis:document"), - VersioningState.MAJOR) - .assertThat().existsInRepo(); - - - assertTrue(waitForMetadataIndexing(file.getName(), true), - "Indexing is not working after the sharding model has been removed"); - - } -}