[MNT-23154] Enable multiple content fields in the same document => Integration Tests + Test Fixes

search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java

@@ -29,6 +29,8 @@ package org.alfresco.solr;
 import static java.util.Arrays.asList;
 import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_DOC_TYPE;
 
+import org.alfresco.model.ContentModel;
+import org.alfresco.service.namespace.QName;
 import org.alfresco.solr.client.Node;
 import org.alfresco.solr.client.NodeMetaData;
 import org.alfresco.solr.client.SOLRAPIQueueClient;
@@ -79,6 +81,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
 
@@ -776,6 +779,23 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer
         //First map the nodes to a transaction.
         SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes);
 
+        //Next map a node to the NodeMetaData
+        int i=0;
+        for(NodeMetaData nodeMetaData : nodeMetaDatas)
+        {
+            SOLRAPIQueueClient.NODE_META_DATA_MAP.put(nodeMetaData.getId(), nodeMetaData);
+            SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), Map.of(ContentModel.PROP_CONTENT, content.get(i++)));
+        }
+
+        //Next add the transaction to the queue
+        SOLRAPIQueueClient.TRANSACTION_QUEUE.add(transaction);
+    }
+
+    public static void indexTransactionWithMultipleContentFields(Transaction transaction, List<Node> nodes, List<NodeMetaData> nodeMetaDatas, List<Map<QName, String>> content)
+    {
+        //First map the nodes to a transaction.
+        SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes);
+
         //Next map a node to the NodeMetaData
         int i=0;
         for(NodeMetaData nodeMetaData : nodeMetaDatas)

search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java

@@ -26,6 +26,7 @@
 
 package org.alfresco.solr;
 
+import org.alfresco.model.ContentModel;
 import org.alfresco.repo.search.impl.parsers.FTSQueryParser;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.cmr.search.SearchParameters;
@@ -85,6 +86,7 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Properties;
 
 import static java.util.Optional.of;
@@ -662,7 +664,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS
         for(NodeMetaData nodeMetaData : nodeMetaDatas)
         {
             SOLRAPIQueueClient.NODE_META_DATA_MAP.put(nodeMetaData.getId(), nodeMetaData);
-            SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), content.get(i++));
+            SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), Map.of(ContentModel.PROP_CONTENT, content.get(i++)));
         }
 
         //Next add the transaction to the queue

search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Search Services
  * %%
- * Copyright (C) 2005 - 2020 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -46,7 +46,7 @@ import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
 @RunWith(MockitoJUnitRunner.class)
-public class ContentTrackerIT
+public class ContentTrackerTest
 {
     private ContentTracker contentTracker;
     

search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java

@@ -0,0 +1,382 @@
+/*
+ * #%L
+ * Alfresco Search Services
+ * %%
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
+ * %%
+ * This file is part of the Alfresco software. 
+ * If the software was purchased under a paid Alfresco license, the terms of 
+ * the paid license agreement will prevail.  Otherwise, the software is 
+ * provided under the following open source license terms:
+ * 
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ * #L%
+ */
+
+package org.alfresco.solr.tracker;
+
+import org.alfresco.model.ContentModel;
+import org.alfresco.service.namespace.QName;
+import org.alfresco.solr.AbstractAlfrescoDistributedIT;
+import org.alfresco.solr.client.Acl;
+import org.alfresco.solr.client.ContentPropertyValue;
+import org.alfresco.solr.client.Node;
+import org.alfresco.solr.client.NodeMetaData;
+import org.alfresco.solr.client.Transaction;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.TermQuery;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static java.util.Collections.singletonList;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.IntStream.range;
+import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME;
+import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
+import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
+import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
+import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
+import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
+import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
+import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
+
+@SolrTestCaseJ4.SuppressSSL
+public class DistributedContentPropertiesIT extends AbstractAlfrescoDistributedIT
+{
+    @BeforeClass
+    public static void initData() throws Throwable
+    {
+        var solrCoreProperties = DEFAULT_CORE_PROPS;
+        solrCoreProperties.setProperty("solr.enableIndexingCustomContent", "true");
+
+        initSolrServers(3, DistributedContentPropertiesIT.class.getSimpleName(), solrCoreProperties);
+    }
+
+    @After
+    public void clearData() throws Exception
+    {
+        deleteByQueryAllClients("*:*");
+        explicitCommitOnAllClients();
+    }
+
+    @AfterClass
+    public static void destroyData()
+    {
+        dismissSolrServers();
+    }
+
+    /**
+     * In this scenario we have n nodes.
+     * Among them:
+     *
+     * <ul>
+     *     <li>n-m have one default cm:content field </li>
+     *     <li>the m do not have a value for the content field (i.e. the value of the content field is empty) </li>
+     * </ul>
+     */
+    @Test
+    public void eachNodeHasAtMaximumOneCmContentField() throws Exception
+    {
+        putHandleDefaults();
+
+        var aclChangeSet = getAclChangeSet(1, 1);
+        var acl = getAcl(aclChangeSet);
+
+        var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
+        indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders));
+
+        var howManyTestNodes = 10;
+
+        var transaction = getTransaction(0, howManyTestNodes);
+
+        var nodes = nodes(howManyTestNodes, transaction, acl);
+        var metadata = metadata(nodes, transaction, acl);
+
+        var howManyNodesWithContent = howManyTestNodes - 3;
+        indexTransaction(transaction, nodes, metadata, textContent(nodes, "Lorem ipsum dolor sit amet", howManyNodesWithContent));
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")),
+                howManyNodesWithContent, MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")),
+                0, MAX_WAIT_TIME);
+    }
+
+    /**
+     * In this scenario we have n nodes.
+     * Among them:
+     *
+     * <ul>
+     *     <li>n-m have one custom content field (i.e. a field different from cm:content).</li>
+     *     <li>the m do not have a value for the content field (i.e. the value of the content field is empty) </li>
+     * </ul>
+     */
+    @Test
+    public void eachNodeHasAtMaximumOneCustomContentField() throws Exception
+    {
+        putHandleDefaults();
+
+        var aclChangeSet = getAclChangeSet(1, 1);
+        var acl = getAcl(aclChangeSet);
+
+        // Arbitrary acl data.
+        var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
+        indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders));
+
+        var howManyTestNodes = 10;
+
+        var transaction = getTransaction(0, howManyTestNodes);
+
+        var nodes = nodes(howManyTestNodes, transaction, acl);
+        var metadata =
+                metadata(nodes, transaction, acl).stream()
+                                .peek(nodeMetadata -> {
+                                    nodeMetadata.getProperties().remove(ContentModel.PROP_CONTENT);
+                                    nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC,
+                                            new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null));})
+                                .collect(toList());
+
+        var howManyNodesWithContent = howManyTestNodes - 4;
+        var howManyNodesWithoutContent = howManyTestNodes - howManyNodesWithContent;
+
+        var textContents =
+                Stream.concat(
+                        range(0, howManyNodesWithContent)
+                                .mapToObj(index -> Map.of(ContentModel.PROP_PERSONDESC, "consectetur Adipiscing elit " + System.currentTimeMillis())),
+                        range(0, howManyNodesWithoutContent)
+                                .mapToObj(index -> Map.of(ContentModel.PROP_PERSONDESC, "")))
+                    .collect(toList());
+
+        indexTransactionWithMultipleContentFields(transaction, nodes, metadata, textContents);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "adipiscing")),
+                howManyNodesWithContent,
+                MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "adipiscing")),
+                0, MAX_WAIT_TIME);
+    }
+
+    /**
+     * In this scenario we have n nodes.
+     * Among them:
+     *
+     * <ul>
+     *     <li>n-m have one default and one custom content field (i.e. a field different from cm:content).</li>
+     *     <li>the m do not have a value for those content fields (i.e. the value of the content fields is empty) </li>
+     * </ul>
+     */
+    @Test
+    public void eachNodeHasOneCustomAndOneDefaultContentField() throws Exception
+    {
+        putHandleDefaults();
+
+        var aclChangeSet = getAclChangeSet(1, 1);
+        var acl = getAcl(aclChangeSet);
+
+        var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
+        indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders));
+
+        var howManyTestNodes = 10;
+
+        var transaction = getTransaction(0, howManyTestNodes);
+
+        var nodes = nodes(howManyTestNodes, transaction, acl);
+        var metadata =
+                metadata(nodes, transaction, acl).stream()
+                        .peek(nodeMetadata ->
+                            nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC,
+                                    new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null)))
+                        .collect(toList());
+
+        var howManyNodesWithContent = howManyTestNodes - 2;
+
+        indexTransactionWithMultipleContentFields(
+                transaction,
+                nodes,
+                metadata,
+                textContentWithMultipleContentFields(
+                        nodes,
+                        "Lorem ipsum dolor sit amet",
+                        "consectetur Adipiscing elit",
+                        howManyNodesWithContent));
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "consectetur")),
+                howManyNodesWithContent,
+                MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")),
+                howManyNodesWithContent, MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")),
+                0,
+                MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "elit")),
+                0, MAX_WAIT_TIME);
+    }
+
+    /**
+     * In this scenario we have n + m + y nodes.
+     * Among them:
+     *
+     * <ul>
+     *     <li>n nodes have one custom content field (i.e. a field different from cm:content).</li>
+     *     <li>m nodes have one default content field (i.e. cm:content).</li>
+     *     <li>y nodes do not have a value for the content field </li>
+     * </ul>
+     */
+    @Test
+    public void someNodeHasOneCustomAndSomeNodeHasOneDefaultContentField() throws Exception
+    {
+        putHandleDefaults();
+
+        var aclChangeSet = getAclChangeSet(1, 1);
+        var acl = getAcl(aclChangeSet);
+
+        var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
+        indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders));
+
+        var howManyTestNodes = 10;
+
+        var transaction = getTransaction(0, howManyTestNodes);
+
+        var nodes = nodes(howManyTestNodes, transaction, acl);
+        var metadata =
+                metadata(nodes, transaction, acl).stream()
+                        .peek(nodeMetadata ->
+                            nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC,
+                                    new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null)))
+                        .collect(toList());
+
+        var howManyNodesWithContent = howManyTestNodes - 2;
+        var howManyNodesWithDefaultContentField = 3;
+        var howManyNodesWithCustomContentField = howManyNodesWithContent - howManyNodesWithDefaultContentField;
+        var howManyNodesWithoutContent = howManyTestNodes - howManyNodesWithContent;
+
+        var baseCmContentText = "Lorem ipsum dolor sit amet";
+        var basePersonDescriptionText = "consectetur Adipiscing elit";
+
+        var texts =
+                Stream.concat(
+                    range(0, howManyNodesWithDefaultContentField)
+                        .mapToObj(index -> Map.of(
+                                            ContentModel.PROP_CONTENT, baseCmContentText + " " + System.currentTimeMillis(),
+                                            ContentModel.PROP_PERSONDESC, "")),
+                    range(0, howManyNodesWithCustomContentField)
+                        .mapToObj(index -> Map.of(
+                                            ContentModel.PROP_CONTENT, "",
+                                            ContentModel.PROP_PERSONDESC, basePersonDescriptionText + " " + System.currentTimeMillis())));
+
+        var textContents =
+                Stream.concat(
+                    texts,
+                    range(0, howManyNodesWithoutContent)
+                        .mapToObj(index -> Map.of(
+                                ContentModel.PROP_CONTENT, "",
+                                ContentModel.PROP_PERSONDESC, ""))).collect(toList());
+
+        indexTransactionWithMultipleContentFields(
+                transaction,
+                nodes,
+                metadata,
+                textContents);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "consectetur")),
+                howManyNodesWithCustomContentField,
+                MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")),
+                howManyNodesWithDefaultContentField, MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")),
+                0,
+                MAX_WAIT_TIME);
+
+        waitForDocCount(
+                new TermQuery(
+                        new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "elit")),
+                0, MAX_WAIT_TIME);
+    }
+
+    private List<Node> nodes(int howMany, Transaction transaction, Acl acl)
+    {
+        return range(0, howMany)
+                .mapToObj(index -> getNode(transaction, acl, Node.SolrApiNodeStatus.UPDATED))
+                .collect(toList());
+    }
+
+    private List<NodeMetaData> metadata(List<Node> nodes, Transaction transaction, Acl acl)
+    {
+        return nodes.stream()
+                .map(node -> getNodeMetaData(node, transaction, acl, "mike", null, false))
+                .collect(toList());
+    }
+
+    private List<String> textContent(List<Node> nodes, String baseText, int limit)
+    {
+        return Stream.concat(
+                nodes.stream()
+                        .map(node -> baseText + " " + System.currentTimeMillis())
+                        .limit(limit),
+                range(0, nodes.size() - limit)
+                        .mapToObj(index -> "")).collect(toList());
+    }
+
+    private List<Map<QName, String>> textContentWithMultipleContentFields(List<Node> nodes, String baseCmContentText, String basePersonDescriptionText, int limit)
+    {
+        var texts = nodes.stream()
+                        .map(node -> Map.of(
+                                        ContentModel.PROP_CONTENT, baseCmContentText + " " + System.currentTimeMillis(),
+                                        ContentModel.PROP_PERSONDESC, basePersonDescriptionText + " " + System.currentTimeMillis()))
+                        .limit(limit);
+
+        return Stream.concat(
+                texts,
+                range(0, nodes.size() - limit)
+                        .mapToObj(index -> Map.of(
+                                ContentModel.PROP_CONTENT, "",
+                                ContentModel.PROP_PERSONDESC, ""))).collect(toList());
+    }
+}

search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Search Services
  * %%
- * Copyright (C) 2005 - 2020 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -62,7 +62,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
     public final static List<Transaction> TRANSACTION_QUEUE = Collections.synchronizedList(new ArrayList<>());
     public final static Map<Long, List<Node>> NODE_MAP = Collections.synchronizedMap(new HashMap<>());
     public final static Map<Long, NodeMetaData> NODE_META_DATA_MAP = Collections.synchronizedMap(new HashMap<>());
-    public final static Map<Long, String> NODE_CONTENT_MAP =  Collections.synchronizedMap(new HashMap<>());
+    public final static Map<Long, Map<QName, String>> NODE_CONTENT_MAP =  Collections.synchronizedMap(new HashMap<>());
 
     private static boolean throwException;
 
@@ -151,7 +151,6 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
                 .collect(toList());
     }
 
-
     public List<AlfrescoModelDiff> getModelsDiff(String coreName, List<AlfrescoModel> currentModels) throws IOException, JSONException
     {
         if(throwException)
@@ -338,7 +337,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient
 
         if(NODE_CONTENT_MAP.containsKey(nodeId))
         {
-            return new GetTextContentResponse(new DummyResponse(NODE_CONTENT_MAP.get(nodeId)));
+            return new GetTextContentResponse(new DummyResponse(NODE_CONTENT_MAP.get(nodeId).get(propertyQName)));
         }
 
         return new GetTextContentResponse(new DummyResponse("Hello world " + nodeId));