From a8059c6f8588017435faa73297d0e1c0075cc44a Mon Sep 17 00:00:00 2001 From: Elia Date: Fri, 26 Aug 2022 16:59:45 +0200 Subject: [PATCH 1/3] [MNT-23154] Allow custom d:content properties to be indexed Added flag in solrcore.properties (cherry picked from commit e9e2c1c64958b74e47021678a3fc35a3075a874b) --- .../alfresco/solr/SolrInformationServer.java | 38 ++++++++++++++++--- .../templates/rerank/conf/solrcore.properties | 5 +++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index 6af6c1036..da9077442 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -361,6 +361,9 @@ public class SolrInformationServer implements InformationServer private static final long CONTENT_OUTDATED_MARKER = -10; private static final long CONTENT_UPDATED_MARKER = -20; + private static final String CONTENT_FIELD_NAME = "contentFieldName"; + + private static final String CONTENT_LOCALE = "contentLocale"; private static final String CONTENT_LOCALE_FIELD = "content@s__locale@{http://www.alfresco.org/model/content/1.0}content"; private static final Set ID_AND_CONTENT_VERSION_ID_AND_CONTENT_LOCALE = new HashSet<>(asList(FIELD_SOLR4_ID, LATEST_APPLIED_CONTENT_VERSION_ID, CONTENT_LOCALE_FIELD)); @@ -460,6 +463,8 @@ public class SolrInformationServer implements InformationServer private final boolean dateFieldDestructuringHasBeenEnabledOnThisInstance; + private final boolean enabledIndexCustomContent; + static class DocListCollector implements Collector, LeafCollector { private final IntArrayList docs = new IntArrayList(); @@ -642,6 +647,8 @@ public class SolrInformationServer implements InformationServer port = portNumber(props); baseUrl = baseUrl(props); + enabledIndexCustomContent = Boolean.parseBoolean(coreConfiguration.getProperty("solr.enableIndexingCustomContent", "false")); + dateFieldDestructuringHasBeenEnabledOnThisInstance = Boolean.parseBoolean(coreConfiguration.getProperty("alfresco.destructureDateFields", "true")); LOGGER.info( "Date fields destructuring has been {} on this instance.", @@ -999,9 +1006,27 @@ public class SolrInformationServer implements InformationServer String idString = id.stringValue(); TenantDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(idString); - ofNullable(document.getField(CONTENT_LOCALE_FIELD)) - .map(IndexableField::stringValue) - .ifPresent(value -> tenantAndDbId.setProperty(CONTENT_LOCALE_FIELD, value)); + if (enabledIndexCustomContent) + { + + document.getFields().stream() + .filter(field -> field.name().startsWith(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX)) + .findFirst() + .ifPresent(field -> { + tenantAndDbId.setProperty(CONTENT_FIELD_NAME, field.name()); + tenantAndDbId.setProperty(CONTENT_LOCALE, field.stringValue()); + }); + } + else + { + ofNullable(document.getField(CONTENT_LOCALE_FIELD)) + .map(IndexableField::stringValue) + .ifPresent(value -> { + tenantAndDbId.setProperty(CONTENT_FIELD_NAME, CONTENT_LOCALE_FIELD); + tenantAndDbId.setProperty(CONTENT_LOCALE, value); + }); + } + tenantAndDbId.setProperty( LATEST_APPLIED_CONTENT_VERSION_ID, @@ -1933,7 +1958,7 @@ public class SolrInformationServer implements InformationServer docRef.tenant, docRef.dbId)); - if (docRef.optionalBag.containsKey(CONTENT_LOCALE_FIELD)) + if (docRef.optionalBag.containsKey(CONTENT_FIELD_NAME)) { addContentToDoc(docRef, doc, docRef.dbId); } @@ -2713,8 +2738,9 @@ public class SolrInformationServer implements InformationServer private void addContentToDoc(TenantDbId docRef, SolrInputDocument doc, long dbId) throws AuthenticationException, IOException { - String locale = (String) docRef.optionalBag.get(CONTENT_LOCALE_FIELD); - String qNamePart = CONTENT_LOCALE_FIELD.substring(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX.length()); + String fieldName = (String) docRef.optionalBag.get(CONTENT_FIELD_NAME); + String locale = (String) docRef.optionalBag.get(CONTENT_LOCALE); + String qNamePart = fieldName.substring(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX.length()); QName propertyQName = QName.createQName(qNamePart); addContentPropertyToDocUsingAlfrescoRepository(doc, propertyQName, dbId, locale); } diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index f512cb9cd..720524a02 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -234,6 +234,11 @@ solr.request.content.compress=false # solr.initial.transaction.range=0-2000 + +# set it to true to allow custom d:content fields to be indexed. +# N.B. At most one content field can be indexed for each document +solr.enableIndexingCustomContent=false + # Facet query limit when retrieving info for the stats (used in the acl transaction and index transaction reports # and fix) alfresco.stats.facetLimit=100 From 64e3827e809cd5ae551a6594bf7caab0bde49b08 Mon Sep 17 00:00:00 2001 From: Andrea Gazzarini Date: Fri, 2 Sep 2022 15:19:19 +0200 Subject: [PATCH 2/3] [MNT-23154] Enable multiple content fields in the same document (cherry picked from commit df990a25721558e08fa826123e8f63570444f7d9) --- .../alfresco/solr/AlfrescoSolrDataModel.java | 27 ++++++++ .../alfresco/solr/SolrInformationServer.java | 64 +++++++++---------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index 293870c94..b3bd9f6d1 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -36,6 +36,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -45,6 +46,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.model.ContentModel; @@ -128,17 +130,42 @@ import static org.alfresco.solr.SolrInformationServer.UNIT_OF_TIME_YEAR_FIELD_SU */ public class AlfrescoSolrDataModel implements QueryConstants { + public static class ContentPropertySpecs { + public final String fieldName; + public final String locale; + + public ContentPropertySpecs(String fieldName, String locale) { + this.fieldName = fieldName; + this.locale = locale; + } + } + public static class TenantDbId { public String tenant; public Long dbId; + private List contentPropertySpecsList; + public Map optionalBag = new HashMap<>(); public void setProperty(String name, Object value) { optionalBag.put(name, value); } + + public boolean hasAtLeastOneContentProperty() { + return contentPropertySpecsList != null && !contentPropertySpecsList.isEmpty(); + } + + public void addContentPropertiesSpecs(List specsList) + { + contentPropertySpecsList = Collections.unmodifiableList(specsList); + } + + public Stream contentPropertySpecsStream() { + return contentPropertySpecsList.stream(); + } } public enum FieldUse diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index da9077442..71809ad60 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -28,8 +28,10 @@ package org.alfresco.solr; import static java.util.Arrays.asList; import static java.util.Arrays.stream; +import static java.util.Collections.emptyList; import static java.util.Optional.empty; import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toList; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_ACLID; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_ACLTXCOMMITTIME; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_ACLTXID; @@ -1006,27 +1008,17 @@ public class SolrInformationServer implements InformationServer String idString = id.stringValue(); TenantDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(idString); - if (enabledIndexCustomContent) - { - - document.getFields().stream() - .filter(field -> field.name().startsWith(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX)) - .findFirst() - .ifPresent(field -> { - tenantAndDbId.setProperty(CONTENT_FIELD_NAME, field.name()); - tenantAndDbId.setProperty(CONTENT_LOCALE, field.stringValue()); - }); - } - else - { - ofNullable(document.getField(CONTENT_LOCALE_FIELD)) - .map(IndexableField::stringValue) - .ifPresent(value -> { - tenantAndDbId.setProperty(CONTENT_FIELD_NAME, CONTENT_LOCALE_FIELD); - tenantAndDbId.setProperty(CONTENT_LOCALE, value); - }); - } - + tenantAndDbId.addContentPropertiesSpecs( + enabledIndexCustomContent + ? document.getFields().stream() + .filter(field -> field.name().startsWith(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX)) + .map(field -> new AlfrescoSolrDataModel.ContentPropertySpecs(field.name(),field.stringValue())) + .collect(toList()) + : ofNullable(document.getField(CONTENT_LOCALE_FIELD)) + .map(IndexableField::stringValue) + .map(value -> new AlfrescoSolrDataModel.ContentPropertySpecs(CONTENT_LOCALE_FIELD, value)) + .map(Collections::singletonList) + .orElse(emptyList())); tenantAndDbId.setProperty( LATEST_APPLIED_CONTENT_VERSION_ID, @@ -1904,7 +1896,7 @@ public class SolrInformationServer implements InformationServer nmdp.setMaxResults(1); // Gets only one Optional> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); - allNodeMetaDatas.addAll(nodeMetaDatas.orElse(Collections.emptyList())); + allNodeMetaDatas.addAll(nodeMetaDatas.orElse(emptyList())); } return allNodeMetaDatas; @@ -1958,7 +1950,7 @@ public class SolrInformationServer implements InformationServer docRef.tenant, docRef.dbId)); - if (docRef.optionalBag.containsKey(CONTENT_FIELD_NAME)) + if (docRef.hasAtLeastOneContentProperty()) { addContentToDoc(docRef, doc, docRef.dbId); } @@ -2010,8 +2002,8 @@ public class SolrInformationServer implements InformationServer categorizeNodes(nodes, nodeIdsToNodes, nodeStatusToNodeIds); List deletedNodeIds = notNullOrEmpty(nodeStatusToNodeIds.get(SolrApiNodeStatus.DELETED)); - List shardDeletedNodeIds = Collections.emptyList(); - List shardUpdatedNodeIds = Collections.emptyList(); + List shardDeletedNodeIds = emptyList(); + List shardUpdatedNodeIds = emptyList(); if (cascadeTrackingEnabled()) { shardDeletedNodeIds = notNullOrEmpty(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED)); @@ -2463,7 +2455,7 @@ public class SolrInformationServer implements InformationServer dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields() .stream() .map(FieldInstance::getField) - .collect(Collectors.toList())); + .collect(toList())); for (PropertyValue singleValue : typedValue.getValues()) { @@ -2736,13 +2728,17 @@ public class SolrInformationServer implements InformationServer } } - private void addContentToDoc(TenantDbId docRef, SolrInputDocument doc, long dbId) throws AuthenticationException, IOException + private void addContentToDoc(TenantDbId docRef, SolrInputDocument doc, long dbId) { - String fieldName = (String) docRef.optionalBag.get(CONTENT_FIELD_NAME); - String locale = (String) docRef.optionalBag.get(CONTENT_LOCALE); - String qNamePart = fieldName.substring(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX.length()); - QName propertyQName = QName.createQName(qNamePart); - addContentPropertyToDocUsingAlfrescoRepository(doc, propertyQName, dbId, locale); + docRef.contentPropertySpecsStream() + .forEach(propertySpecs -> { + try { + var propertyQName = QName.createQName(propertySpecs.fieldName.substring(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX.length())); + addContentPropertyToDocUsingAlfrescoRepository(doc, propertyQName, dbId, propertySpecs.locale); + } catch (AuthenticationException | IOException exception) { + throw new RuntimeException(exception); + } + }); } @@ -2855,7 +2851,7 @@ public class SolrInformationServer implements InformationServer { if (mlTextPropertyValue == null) { - return Collections.emptyList(); + return emptyList(); } List values = new ArrayList<>(); @@ -2917,7 +2913,7 @@ public class SolrInformationServer implements InformationServer .stream() .filter(Objects::nonNull) .map(mlTextPropertyValue::getValue) - .collect(Collectors.toList()); + .collect(toList()); if (!localisedValues.isEmpty()) { From 177455b925cda0677318f7eedc0fb47aebb4c8af Mon Sep 17 00:00:00 2001 From: Andrea Gazzarini Date: Tue, 6 Sep 2022 16:10:13 +0200 Subject: [PATCH 3/3] [MNT-23154] Enable multiple content fields in the same document => Integration Tests + Test Fixes (cherry picked from commit 4ef395fefddcac2e9e0e4e158bb7e972fa4d37e5) --- .../solr/AbstractAlfrescoDistributedIT.java | 20 + .../alfresco/solr/AbstractAlfrescoSolrIT.java | 4 +- ...TrackerIT.java => ContentTrackerTest.java} | 4 +- .../DistributedContentPropertiesIT.java | 382 ++++++++++++++++++ .../solr/client/SOLRAPIQueueClient.java | 7 +- 5 files changed, 410 insertions(+), 7 deletions(-) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/{ContentTrackerIT.java => ContentTrackerTest.java} (98%) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 552e2416a..64fe10546 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -29,6 +29,8 @@ package org.alfresco.solr; import static java.util.Arrays.asList; import static org.alfresco.repo.search.adaptor.QueryConstants.FIELD_DOC_TYPE; +import org.alfresco.model.ContentModel; +import org.alfresco.service.namespace.QName; import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIQueueClient; @@ -79,6 +81,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Random; @@ -776,6 +779,23 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer //First map the nodes to a transaction. SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes); + //Next map a node to the NodeMetaData + int i=0; + for(NodeMetaData nodeMetaData : nodeMetaDatas) + { + SOLRAPIQueueClient.NODE_META_DATA_MAP.put(nodeMetaData.getId(), nodeMetaData); + SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), Map.of(ContentModel.PROP_CONTENT, content.get(i++))); + } + + //Next add the transaction to the queue + SOLRAPIQueueClient.TRANSACTION_QUEUE.add(transaction); + } + + public static void indexTransactionWithMultipleContentFields(Transaction transaction, List nodes, List nodeMetaDatas, List> content) + { + //First map the nodes to a transaction. + SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes); + //Next map a node to the NodeMetaData int i=0; for(NodeMetaData nodeMetaData : nodeMetaDatas) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java index d73d2638b..3d8027de8 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoSolrIT.java @@ -26,6 +26,7 @@ package org.alfresco.solr; +import org.alfresco.model.ContentModel; import org.alfresco.repo.search.impl.parsers.FTSQueryParser; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.cmr.search.SearchParameters; @@ -85,6 +86,7 @@ import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; import static java.util.Optional.of; @@ -662,7 +664,7 @@ public abstract class AbstractAlfrescoSolrIT implements SolrTestFiles, AlfrescoS for(NodeMetaData nodeMetaData : nodeMetaDatas) { SOLRAPIQueueClient.NODE_META_DATA_MAP.put(nodeMetaData.getId(), nodeMetaData); - SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), content.get(i++)); + SOLRAPIQueueClient.NODE_CONTENT_MAP.put(nodeMetaData.getId(), Map.of(ContentModel.PROP_CONTENT, content.get(i++))); } //Next add the transaction to the queue diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java index c574311d1..d37dc75d4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Search Services * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -46,7 +46,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ContentTrackerIT +public class ContentTrackerTest { private ContentTracker contentTracker; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java new file mode 100644 index 000000000..358122b8c --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedContentPropertiesIT.java @@ -0,0 +1,382 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr.tracker; + +import org.alfresco.model.ContentModel; +import org.alfresco.service.namespace.QName; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.ContentPropertyValue; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.Transaction; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static java.util.stream.IntStream.range; +import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; + +@SolrTestCaseJ4.SuppressSSL +public class DistributedContentPropertiesIT extends AbstractAlfrescoDistributedIT +{ + @BeforeClass + public static void initData() throws Throwable + { + var solrCoreProperties = DEFAULT_CORE_PROPS; + solrCoreProperties.setProperty("solr.enableIndexingCustomContent", "true"); + + initSolrServers(3, DistributedContentPropertiesIT.class.getSimpleName(), solrCoreProperties); + } + + @After + public void clearData() throws Exception + { + deleteByQueryAllClients("*:*"); + explicitCommitOnAllClients(); + } + + @AfterClass + public static void destroyData() + { + dismissSolrServers(); + } + + /** + * In this scenario we have n nodes. + * Among them: + * + *
    + *
  • n-m have one default cm:content field
    • + *
  • the m do not have a value for the content field (i.e. the value of the content field is empty)
    • + *
+ */ + @Test + public void eachNodeHasAtMaximumOneCmContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = metadata(nodes, transaction, acl); + + var howManyNodesWithContent = howManyTestNodes - 3; + indexTransaction(transaction, nodes, metadata, textContent(nodes, "Lorem ipsum dolor sit amet", howManyNodesWithContent)); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")), + howManyNodesWithContent, MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")), + 0, MAX_WAIT_TIME); + } + + /** + * In this scenario we have n nodes. + * Among them: + * + *
    + *
  • n-m have one custom content field (i.e. a field different from cm:content).
    • + *
  • the m do not have a value for the content field (i.e. the value of the content field is empty)
    • + *
+ */ + @Test + public void eachNodeHasAtMaximumOneCustomContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + // Arbitrary acl data. + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = + metadata(nodes, transaction, acl).stream() + .peek(nodeMetadata -> { + nodeMetadata.getProperties().remove(ContentModel.PROP_CONTENT); + nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC, + new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null));}) + .collect(toList()); + + var howManyNodesWithContent = howManyTestNodes - 4; + var howManyNodesWithoutContent = howManyTestNodes - howManyNodesWithContent; + + var textContents = + Stream.concat( + range(0, howManyNodesWithContent) + .mapToObj(index -> Map.of(ContentModel.PROP_PERSONDESC, "consectetur Adipiscing elit " + System.currentTimeMillis())), + range(0, howManyNodesWithoutContent) + .mapToObj(index -> Map.of(ContentModel.PROP_PERSONDESC, ""))) + .collect(toList()); + + indexTransactionWithMultipleContentFields(transaction, nodes, metadata, textContents); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "adipiscing")), + howManyNodesWithContent, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "adipiscing")), + 0, MAX_WAIT_TIME); + } + + /** + * In this scenario we have n nodes. + * Among them: + * + *
    + *
  • n-m have one default and one custom content field (i.e. a field different from cm:content).
    • + *
  • the m do not have a value for those content fields (i.e. the value of the content fields is empty)
    • + *
+ */ + @Test + public void eachNodeHasOneCustomAndOneDefaultContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = + metadata(nodes, transaction, acl).stream() + .peek(nodeMetadata -> + nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC, + new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null))) + .collect(toList()); + + var howManyNodesWithContent = howManyTestNodes - 2; + + indexTransactionWithMultipleContentFields( + transaction, + nodes, + metadata, + textContentWithMultipleContentFields( + nodes, + "Lorem ipsum dolor sit amet", + "consectetur Adipiscing elit", + howManyNodesWithContent)); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "consectetur")), + howManyNodesWithContent, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")), + howManyNodesWithContent, MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")), + 0, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "elit")), + 0, MAX_WAIT_TIME); + } + + /** + * In this scenario we have n + m + y nodes. + * Among them: + * + *
    + *
  • n nodes have one custom content field (i.e. a field different from cm:content).
    • + *
  • m nodes have one default content field (i.e. cm:content).
    • + *
  • y nodes do not have a value for the content field
    • + *
+ */ + @Test + public void someNodeHasOneCustomAndSomeNodeHasOneDefaultContentField() throws Exception + { + putHandleDefaults(); + + var aclChangeSet = getAclChangeSet(1, 1); + var acl = getAcl(aclChangeSet); + + var aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + indexAclChangeSet(aclChangeSet, singletonList(acl), singletonList(aclReaders)); + + var howManyTestNodes = 10; + + var transaction = getTransaction(0, howManyTestNodes); + + var nodes = nodes(howManyTestNodes, transaction, acl); + var metadata = + metadata(nodes, transaction, acl).stream() + .peek(nodeMetadata -> + nodeMetadata.getProperties().put(ContentModel.PROP_PERSONDESC, + new ContentPropertyValue(Locale.US, 0L, "UTF-8", "text/plain", null))) + .collect(toList()); + + var howManyNodesWithContent = howManyTestNodes - 2; + var howManyNodesWithDefaultContentField = 3; + var howManyNodesWithCustomContentField = howManyNodesWithContent - howManyNodesWithDefaultContentField; + var howManyNodesWithoutContent = howManyTestNodes - howManyNodesWithContent; + + var baseCmContentText = "Lorem ipsum dolor sit amet"; + var basePersonDescriptionText = "consectetur Adipiscing elit"; + + var texts = + Stream.concat( + range(0, howManyNodesWithDefaultContentField) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, baseCmContentText + " " + System.currentTimeMillis(), + ContentModel.PROP_PERSONDESC, "")), + range(0, howManyNodesWithCustomContentField) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, "", + ContentModel.PROP_PERSONDESC, basePersonDescriptionText + " " + System.currentTimeMillis()))); + + var textContents = + Stream.concat( + texts, + range(0, howManyNodesWithoutContent) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, "", + ContentModel.PROP_PERSONDESC, ""))).collect(toList()); + + indexTransactionWithMultipleContentFields( + transaction, + nodes, + metadata, + textContents); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "consectetur")), + howManyNodesWithCustomContentField, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "ipsum")), + howManyNodesWithDefaultContentField, MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}" + ContentModel.PROP_PERSONDESC.getLocalName(), "ipsum")), + 0, + MAX_WAIT_TIME); + + waitForDocCount( + new TermQuery( + new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "elit")), + 0, MAX_WAIT_TIME); + } + + private List nodes(int howMany, Transaction transaction, Acl acl) + { + return range(0, howMany) + .mapToObj(index -> getNode(transaction, acl, Node.SolrApiNodeStatus.UPDATED)) + .collect(toList()); + } + + private List metadata(List nodes, Transaction transaction, Acl acl) + { + return nodes.stream() + .map(node -> getNodeMetaData(node, transaction, acl, "mike", null, false)) + .collect(toList()); + } + + private List textContent(List nodes, String baseText, int limit) + { + return Stream.concat( + nodes.stream() + .map(node -> baseText + " " + System.currentTimeMillis()) + .limit(limit), + range(0, nodes.size() - limit) + .mapToObj(index -> "")).collect(toList()); + } + + private List> textContentWithMultipleContentFields(List nodes, String baseCmContentText, String basePersonDescriptionText, int limit) + { + var texts = nodes.stream() + .map(node -> Map.of( + ContentModel.PROP_CONTENT, baseCmContentText + " " + System.currentTimeMillis(), + ContentModel.PROP_PERSONDESC, basePersonDescriptionText + " " + System.currentTimeMillis())) + .limit(limit); + + return Stream.concat( + texts, + range(0, nodes.size() - limit) + .mapToObj(index -> Map.of( + ContentModel.PROP_CONTENT, "", + ContentModel.PROP_PERSONDESC, ""))).collect(toList()); + } +} \ No newline at end of file diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java index ded8cd45f..ab0abec3a 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIQueueClient.java @@ -2,7 +2,7 @@ * #%L * Alfresco Search Services * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -65,7 +65,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient public final static List TRANSACTION_QUEUE = Collections.synchronizedList(new ArrayList<>()); public final static Map> NODE_MAP = Collections.synchronizedMap(new HashMap<>()); public final static Map NODE_META_DATA_MAP = Collections.synchronizedMap(new HashMap<>()); - public final static Map NODE_CONTENT_MAP = Collections.synchronizedMap(new HashMap<>()); + public final static Map> NODE_CONTENT_MAP = Collections.synchronizedMap(new HashMap<>()); private static boolean throwException; @@ -154,7 +154,6 @@ public class SOLRAPIQueueClient extends SOLRAPIClient .collect(toList()); } - public List getModelsDiff(String coreName, List currentModels) throws IOException, JSONException { if(throwException) @@ -365,7 +364,7 @@ public class SOLRAPIQueueClient extends SOLRAPIClient if(NODE_CONTENT_MAP.containsKey(nodeId)) { - return new GetTextContentResponse(new DummyResponse(NODE_CONTENT_MAP.get(nodeId))); + return new GetTextContentResponse(new DummyResponse(NODE_CONTENT_MAP.get(nodeId).get(propertyQName))); } return new GetTextContentResponse(new DummyResponse("Hello world " + nodeId));