diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 552e2416a..64fe10546 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -29,6 +29,8 @@ package org.alfresco.solr; import static java.util.Arrays.asList; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_DOC_TYPE; +import org.alfresco.model.ContentModel; +import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIQueueClient; @@ -79,6 +81,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Random; @@ -776,6 +779,23 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer //First map the nodes to a transaction. SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes); + //Next map a node to the NodeMetaData + int i=0; + for(NodeMetaData nodeMetaData : nodeMetaDatas) + { + SOLRAPIQueueClient.NODE_META_DATA_MAP.put(nodeMetaData.getId(), nodeMetaData); + SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), Map.of(ContentModel.PROP_CONTENT, content.get(i++))); + } + + //Next add the transaction to the queue + SOLRAPIQueueClient.TRANSACTION_QUEUE.add(transaction); + } + + public static void indexTransactionWithMultipleContentFields(Transaction transaction, List nodes, List nodeMetaDatas, List> content) + { + //First map the nodes to a transaction. + SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes); + //Next map a node to the NodeMetaData int i=0; for(NodeMetaData nodeMetaData : nodeMetaDatas) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java index d73d2638b..3d8027de8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java @@ -26,6 +26,7 @@ package org.alfresco.solr; +import org.alfresco.model.ContentModel; import org.alfresco.repo.search.impl.parsers.FTSQueryParser; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.search.SearchParameters; @@ -85,6 +86,7 @@ import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; import static java.util.Optional.of; @@ -662,7 +664,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS for(NodeMetaData nodeMetaData : nodeMetaDatas) { SOLRAPIQueueClient.NODE_META_DATA_MAP.put(nodeMetaData.getId(), nodeMetaData); - SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), content.get(i++)); + SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), Map.of(ContentModel.PROP_CONTENT, content.get(i++))); } //Next add the transaction to the queue diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java index c574311d1..d37dc75d4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Search Services * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -46,7 +46,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ContentTrackerIT +public class ContentTrackerTest { private ContentTracker contentTracker; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java new file mode 100644 index 000000000..358122b8c --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java @@ -0,0 +1,382 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr.tracker; + +import org.alfresco.model.ContentModel; +import org.alfresco.service.namespace.QName; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.ContentPropertyValue; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.Transaction; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static java.util.stream.IntStream.range; +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; + +@SolrTestCaseJ4.SuppressSSL +public class DistributedContentPropertiesIT extends AbstractAlfrescoDistributedIT +{ + @BeforeClass + public static void initData() throws Throwable + { + var solrCoreProperties = DEFAULT_CORE_PROPS; + solrCoreProperties.setProperty("solr.enableIndexingCustomContent", "true"); + + initSolrServers(3, DistributedContentPropertiesIT.class.getSimpleName(), solrCoreProperties); + } + + @After + public void clearData() throws Exception + { + deleteByQueryAllClients("*:*"); + explicitCommitOnAllClients(); + } + + @AfterClass + public static void destroyData() + { + dismissSolrServers(); + } + + /** + * In this scenario we have n nodes. + * Among them: + * + *
    + *
  • n-m have one default cm:content field
    • + *
  • the m do not have a value for the content field (i.e. the value of the content field is empty)
    • + *
+ */ + @Test + public void eachNodeHasAtMaximumOneCmContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = metadata(nodes, transaction, acl); + + var howManyNodesWithContent = howManyTestNodes - 3; + indexTransaction(transaction, nodes, metadata, textContent(nodes, "Lorem ipsum dolor sit amet", howManyNodesWithContent)); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")), + howManyNodesWithContent, MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")), + 0, MAX_WAIT_TIME); + } + + /** + * In this scenario we have n nodes. + * Among them: + * + *
    + *
  • n-m have one custom content field (i.e. a field different from cm:content).
    • + *
  • the m do not have a value for the content field (i.e. the value of the content field is empty)
    • + *
+ */ + @Test + public void eachNodeHasAtMaximumOneCustomContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + // Arbitrary acl data. + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = + metadata(nodes, transaction, acl).stream() + .peek(nodeMetadata -> { + nodeMetadata.getProperties().remove(ContentModel.PROP_CONTENT); + nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC, + new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null));}) + .collect(toList()); + + var howManyNodesWithContent = howManyTestNodes - 4; + var howManyNodesWithoutContent = howManyTestNodes - howManyNodesWithContent; + + var textContents = + Stream.concat( + range(0, howManyNodesWithContent) + .mapToObj(index -> Map.of(ContentModel.PROP_PERSONDESC, "consectetur Adipiscing elit " + System.currentTimeMillis())), + range(0, howManyNodesWithoutContent) + .mapToObj(index -> Map.of(ContentModel.PROP_PERSONDESC, ""))) + .collect(toList()); + + indexTransactionWithMultipleContentFields(transaction, nodes, metadata, textContents); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "adipiscing")), + howManyNodesWithContent, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "adipiscing")), + 0, MAX_WAIT_TIME); + } + + /** + * In this scenario we have n nodes. + * Among them: + * + *
    + *
  • n-m have one default and one custom content field (i.e. a field different from cm:content).
    • + *
  • the m do not have a value for those content fields (i.e. the value of the content fields is empty)
    • + *
+ */ + @Test + public void eachNodeHasOneCustomAndOneDefaultContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = + metadata(nodes, transaction, acl).stream() + .peek(nodeMetadata -> + nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC, + new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null))) + .collect(toList()); + + var howManyNodesWithContent = howManyTestNodes - 2; + + indexTransactionWithMultipleContentFields( + transaction, + nodes, + metadata, + textContentWithMultipleContentFields( + nodes, + "Lorem ipsum dolor sit amet", + "consectetur Adipiscing elit", + howManyNodesWithContent)); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "consectetur")), + howManyNodesWithContent, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")), + howManyNodesWithContent, MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")), + 0, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "elit")), + 0, MAX_WAIT_TIME); + } + + /** + * In this scenario we have n + m + y nodes. + * Among them: + * + *
    + *
  • n nodes have one custom content field (i.e. a field different from cm:content).
    • + *
  • m nodes have one default content field (i.e. cm:content).
    • + *
  • y nodes do not have a value for the content field
    • + *
+ */ + @Test + public void someNodeHasOneCustomAndSomeNodeHasOneDefaultContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = + metadata(nodes, transaction, acl).stream() + .peek(nodeMetadata -> + nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC, + new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null))) + .collect(toList()); + + var howManyNodesWithContent = howManyTestNodes - 2; + var howManyNodesWithDefaultContentField = 3; + var howManyNodesWithCustomContentField = howManyNodesWithContent - howManyNodesWithDefaultContentField; + var howManyNodesWithoutContent = howManyTestNodes - howManyNodesWithContent; + + var baseCmContentText = "Lorem ipsum dolor sit amet"; + var basePersonDescriptionText = "consectetur Adipiscing elit"; + + var texts = + Stream.concat( + range(0, howManyNodesWithDefaultContentField) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, baseCmContentText + " " + System.currentTimeMillis(), + ContentModel.PROP_PERSONDESC, "")), + range(0, howManyNodesWithCustomContentField) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, "", + ContentModel.PROP_PERSONDESC, basePersonDescriptionText + " " + System.currentTimeMillis()))); + + var textContents = + Stream.concat( + texts, + range(0, howManyNodesWithoutContent) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, "", + ContentModel.PROP_PERSONDESC, ""))).collect(toList()); + + indexTransactionWithMultipleContentFields( + transaction, + nodes, + metadata, + textContents); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "consectetur")), + howManyNodesWithCustomContentField, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")), + howManyNodesWithDefaultContentField, MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")), + 0, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "elit")), + 0, MAX_WAIT_TIME); + } + + private List nodes(int howMany, Transaction transaction, Acl acl) + { + return range(0, howMany) + .mapToObj(index -> getNode(transaction, acl, Node.SolrApiNodeStatus.UPDATED)) + .collect(toList()); + } + + private List metadata(List nodes, Transaction transaction, Acl acl) + { + return nodes.stream() + .map(node -> getNodeMetaData(node, transaction, acl, "mike", null, false)) + .collect(toList()); + } + + private List textContent(List nodes, String baseText, int limit) + { + return Stream.concat( + nodes.stream() + .map(node -> baseText + " " + System.currentTimeMillis()) + .limit(limit), + range(0, nodes.size() - limit) + .mapToObj(index -> "")).collect(toList()); + } + + private List> textContentWithMultipleContentFields(List nodes, String baseCmContentText, String basePersonDescriptionText, int limit) + { + var texts = nodes.stream() + .map(node -> Map.of( + ContentModel.PROP_CONTENT, baseCmContentText + " " + System.currentTimeMillis(), + ContentModel.PROP_PERSONDESC, basePersonDescriptionText + " " + System.currentTimeMillis())) + .limit(limit); + + return Stream.concat( + texts, + range(0, nodes.size() - limit) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, "", + ContentModel.PROP_PERSONDESC, ""))).collect(toList()); + } +} \ No newline at end of file diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index 04d45173a..e69267fa5 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -2,7 +2,7 @@ * #%L * Alfresco Search Services * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -62,7 +62,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient public final static List TRANSACTION_QUEUE = Collections.synchronizedList(new ArrayList<>()); public final static Map> NODE_MAP = Collections.synchronizedMap(new HashMap<>()); public final static Map NODE_META_DATA_MAP = Collections.synchronizedMap(new HashMap<>()); - public final static Map NODE_CONTENT_MAP = Collections.synchronizedMap(new HashMap<>()); + public final static Map> NODE_CONTENT_MAP = Collections.synchronizedMap(new HashMap<>()); private static boolean throwException; @@ -151,7 +151,6 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .collect(toList()); } - public List getModelsDiff(String coreName, List currentModels) throws IOException, JSONException { if(throwException) @@ -338,7 +337,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient if(NODE_CONTENT_MAP.containsKey(nodeId)) { - return new GetTextContentResponse(new DummyResponse(NODE_CONTENT_MAP.get(nodeId))); + return new GetTextContentResponse(new DummyResponse(NODE_CONTENT_MAP.get(nodeId).get(propertyQName))); } return new GetTextContentResponse(new DummyResponse("Hello world " + nodeId));