Merge branch 'feature/SEARCH-1689' into 'feature/SEARCH_1687_ContentStore_Removal'

Feature/search 1689

See merge request search_discovery/insightengine!319

search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformer.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2005-2020 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.solr.transformer;
+
+import org.alfresco.solr.AlfrescoSolrDataModel;
+import org.apache.lucene.index.IndexableField;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.response.DocsStreamer;
+import org.apache.solr.response.ResultContext;
+import org.apache.solr.response.transform.DocTransformer;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SolrReturnFields;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+/**
+ * @author Andy, Elia
+ *
+ */
+public class AlfrescoFieldMapperTransformer extends DocTransformer
+{
+    protected final static Logger log = LoggerFactory.getLogger(AlfrescoFieldMapperTransformer.class);
+
+    private ResultContext context;
+    private SolrReturnFields solrReturnFields;
+    
+    /* (non-Javadoc)
+     * @see org.apache.solr.response.transform.DocTransformer#getName()
+     */
+    @Override
+    public String getName()
+    {
+        return "alfrescoMapper";
+    }
+    
+    
+    public void setContext( ResultContext context ) 
+    {
+        this.context = context;
+    }
+    
+    /* (non-Javadoc)
+     * @see org.apache.solr.response.transform.DocTransformer#transform(org.apache.solr.common.SolrDocument, int)
+     */
+    @Override
+    public void transform(SolrDocument doc, int docid, float score) {
+        Collection<String> fieldNames = new ArrayList<>(doc.getFieldNames());
+        solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest());
+
+        for (String fieldName : fieldNames)
+        {
+           SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName);
+           if(schemaField != null)
+           {
+               String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName);
+               if (isRequestedField(alfrescoFieldName) || alfrescoFieldName.equals("id"))
+               {
+                   Object value = doc.getFieldValue(fieldName);
+                   doc.removeFields(fieldName);
+                   if (schemaField.multiValued())
+                   {
+                       Object collectionValue = ((Collection<Object>) value).stream()
+                               .map(elem -> getFieldValue(schemaField, elem))
+                               .collect(Collectors.toSet());
+                       doc.setField(alfrescoFieldName, collectionValue);
+                   }
+                   else
+                   {
+                       doc.setField(transformToUnderscoreNotation(alfrescoFieldName),
+                               getFieldValue(schemaField, value));
+                   }
+
+               }
+               else
+               {
+                   doc.removeFields(alfrescoFieldName);
+                   doc.removeFields(fieldName);
+               }
+           }
+        }
+    }
+
+    private boolean isRequestedField(String fieldName)
+    {
+        return solrReturnFields.wantsField(transformToUnderscoreNotation(fieldName));
+    }
+
+    private static String transformToUnderscoreNotation(String value)
+    {
+        return value.contains(":") ? value.replace(":", "_") : value;
+    }
+
+    private String removeLocale(String value)
+    {
+        int start = value.lastIndexOf('\u0000');
+        if(start == -1){
+            return value;
+        } else {
+            return value.substring(start + 1);
+        }
+    }
+
+    private Object getFieldValue(SchemaField schemaField, Object value){
+        Object indexedValue = DocsStreamer.getValue(schemaField, (IndexableField) value);
+        return indexedValue instanceof String? removeLocale((String) indexedValue) : indexedValue;
+    }
+}

search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerFactory.java

@@ -27,7 +27,7 @@ import org.apache.solr.response.transform.TransformerFactory;
  * @author Andy
  *
  */
-public class CachedDocTransformerFactory extends TransformerFactory
+public class AlfrescoFieldMapperTransformerFactory extends TransformerFactory
 {
 
     /* (non-Javadoc)
@@ -36,7 +36,7 @@ public class CachedDocTransformerFactory extends TransformerFactory
     @Override
     public DocTransformer create(String field, SolrParams params, SolrQueryRequest req)
     {
-        return new CachedDocTransformer();
+        return new AlfrescoFieldMapperTransformer();
     }
 
 }

search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java

@@ -1,197 +0,0 @@
-/*
- * Copyright (C) 2005-2014 Alfresco Software Limited.
- *
- * This file is part of Alfresco
- *
- * Alfresco is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * Alfresco is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
- */
-package org.alfresco.solr.transformer;
-
-import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.alfresco.solr.AlfrescoCoreAdminHandler;
-import org.alfresco.solr.AlfrescoSolrDataModel;
-import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId;
-import org.alfresco.solr.SolrInformationServer;
-import org.alfresco.solr.content.SolrContentStore;
-import org.apache.lucene.index.IndexableField;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.response.ResultContext;
-import org.apache.solr.response.transform.DocTransformer;
-import org.apache.solr.schema.SchemaField;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author Andy
- *
- */
-public class CachedDocTransformer extends DocTransformer
-{
-    protected final static Logger log = LoggerFactory.getLogger(CachedDocTransformer.class);
-
-    private ResultContext context;
-    
-    /* (non-Javadoc)
-     * @see org.apache.solr.response.transform.DocTransformer#getName()
-     */
-    @Override
-    public String getName()
-    {
-        return "Alfresco cached document transformer";
-    }
-    
-    
-    public void setContext( ResultContext context ) 
-    {
-        this.context = context;
-    }
-    
-    /* (non-Javadoc)
-     * @see org.apache.solr.response.transform.DocTransformer#transform(org.apache.solr.common.SolrDocument, int)
-     */
-    @Override
-    public void transform(SolrDocument doc, int docid, float score) throws IOException
-    {
-        SolrInputDocument cachedDoc = null;
-        try
-        {
-            String id = getFieldValueString(doc, FIELD_SOLR4_ID);
-            TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(id);
-            CoreContainer coreContainer = context.getSearcher().getCore().getCoreContainer();
-            AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler();
-            SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers().get(context.getSearcher().getCore().getName());
-            SolrContentStore solrContentStore = srv.getSolrContentStore();
-            cachedDoc = solrContentStore.retrieveDocFromSolrContentStore(tenantAndDbId.tenant, tenantAndDbId.dbId);
-        }
-        catch(StringIndexOutOfBoundsException e)
-        {
-            // ignore invalid forms ....
-        }
-        
-        if(cachedDoc != null)
-        {
-            Collection<String> fieldNames = cachedDoc.getFieldNames();
-            for (String fieldName : fieldNames)
-            {
-               SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName);
-               if(schemaField != null)
-               {
-                   doc.removeFields(fieldName);
-                   if(schemaField.multiValued())
-                   {
-                       int index = fieldName.lastIndexOf("@{");
-                       if(index == -1)
-                       { 
-                           doc.addField(fieldName, cachedDoc.getFieldValues(fieldName));
-                       }
-                       else
-                       {
-                           String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName);
-                           Collection<Object> values = cachedDoc.getFieldValues(fieldName);
-
-                           //Guard against null pointer in case data model field name does not match up with cachedDoc field name.
-                           if(values != null) {
-                               ArrayList<Object> newValues = new ArrayList<Object>(values.size());
-                               for (Object value : values) {
-                                   if (value instanceof String) {
-                                       String stringValue = (String) value;
-                                       int start = stringValue.lastIndexOf('\u0000');
-                                       if (start == -1) {
-                                           newValues.add(stringValue);
-                                       } else {
-                                           newValues.add(stringValue.substring(start + 1));
-                                       }
-                                   } else {
-                                       newValues.add(value);
-                                   }
-
-                               }
-                               doc.removeFields(alfrescoFieldName);
-                               doc.addField(alfrescoFieldName, newValues);
-                           }
-                       }
-                   }
-                   else
-                   {
-                       int index = fieldName.lastIndexOf("@{");
-                       if(index == -1)
-                       { 
-                            doc.addField(fieldName, cachedDoc.getFieldValue(fieldName));
-                       }
-                       else
-                       {
-                           String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName);
-                           alfrescoFieldName = alfrescoFieldName.contains(":") ? alfrescoFieldName.replace(":", "_") : alfrescoFieldName;
-                           Object value = cachedDoc.getFieldValue(fieldName);
-                           if(value instanceof String)
-                           {
-                               String stringValue = (String) value;
-                               int start = stringValue.lastIndexOf('\u0000');
-                               if(start == -1)
-                               {
-                                   doc.removeFields(alfrescoFieldName);
-                                   doc.addField(alfrescoFieldName, stringValue);
-                               }
-                               else
-                               {
-                                   doc.removeFields(alfrescoFieldName);
-                                   doc.addField(alfrescoFieldName, stringValue.substring(start+1));
-                               }
-                           }
-                           else
-                           {
-                               doc.removeFields(alfrescoFieldName); 
-                               doc.addField(alfrescoFieldName, value);
-                           }
-                       }
-                   }
-               }
-            }
-        }
-
-    }
-
-    
-    private String getFieldValueString(SolrDocument doc, String fieldName)
-    {
-        Object o = doc.getFieldValue(fieldName);
-        if(o != null)
-        {
-            if(o instanceof IndexableField)
-            {
-                IndexableField field = (IndexableField)o;
-                return  field.stringValue();
-            }
-            else if(o instanceof String)
-            {
-                return (String)o;
-            }
-            else
-            {
-                return o.toString();
-            }
-        }
-        else
-        {
-            return null;
-        }
-    }
-}

search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java

@@ -18,6 +18,12 @@
  */
 package org.apache.solr.handler.component;
 
+import static java.util.Optional.of;
+import static java.util.Optional.ofNullable;
+import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FACET;
+import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FTS;
+import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.ID;
+import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.SORT;
 import static org.apache.solr.common.params.CommonParams.PATH;
 
 import java.io.BufferedReader;
@@ -28,9 +34,13 @@ import java.io.Reader;
 import java.io.StringWriter;
 import java.lang.invoke.MethodHandles;
 import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.alfresco.error.AlfrescoRuntimeException;
+import org.alfresco.solr.AlfrescoSolrDataModel;
 import org.alfresco.solr.query.AbstractQParser;
+import org.apache.cxf.transport.http.auth.HttpAuthHeader;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.ExitableDirectoryReader;
 import org.apache.lucene.index.IndexableField;
@@ -58,6 +68,7 @@ import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
 import org.apache.solr.search.SolrQueryTimeoutImpl;
+import org.apache.solr.search.SolrReturnFields;
 import org.apache.solr.search.facet.FacetModule;
 import org.apache.solr.util.RTimerTree;
 import org.apache.solr.util.SolrPluginUtils;
@@ -287,11 +298,91 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements
 		return shardHandler;
 	}
 
+
+	/**
+	 * Transform the fieldlist depending on the use of cached transformer:
+	 * [cached] -> add to the field list the translations of the fiels to the internal schema notation
+	 * otherwise -> modify the field list in order to contains a subset of the following fields:
+	 * 		id, DBID, _version_ and score
+	 * @param req
+	 */
+	private void transformFieldList(SolrQueryRequest req)
+	{
+		if (req.getParams().get("originalFl") != null)
+			return;
+
+		Set<String> fieldListSet = new HashSet<>();
+
+		Set<String> defaultNonCachedFields = Set.of("id","DBID", "_version_");
+		Set<String> allowedNonCachedFields = new HashSet<>(defaultNonCachedFields);
+		allowedNonCachedFields.add("score");
+
+		SolrReturnFields solrReturnFields = new SolrReturnFields(req);
+		String originalFieldList = req.getParams().get("fl");
+
+		boolean cacheTransformer = ofNullable(solrReturnFields.getTransformer())
+				.map(t -> t.getName())
+				.map(name -> name.contains("alfrescoMapper"))
+				.orElse(false);
+
+		ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
+
+
+		// In case cache transformer is no set, we need to modify the field list in order return
+		// only id, DBID and _version_ fields
+		if (!cacheTransformer){
+			if (solrReturnFields.wantsAllFields())
+			{
+				fieldListSet.addAll(defaultNonCachedFields);
+			}
+			else
+			{
+				fieldListSet.addAll(solrReturnFields.getLuceneFieldNames()
+						.stream()
+						.filter(field -> allowedNonCachedFields.contains(field))
+						.collect(Collectors.toSet()));
+			}
+
+			params.set("fl", fieldListSet.stream().collect(Collectors.joining(",")));
+		}
+		else
+		{
+			if (solrReturnFields.wantsAllFields() || solrReturnFields.hasPatternMatching())
+			{
+				fieldListSet.add("*");
+			}
+			else
+			{
+				List<AlfrescoSolrDataModel.FieldUse> fieldUsed = List.of(FTS, FACET, ID, SORT);
+				fieldListSet.addAll(solrReturnFields.getLuceneFieldNames().stream()
+						.flatMap(field ->
+								fieldUsed.stream()
+										.map( fieldUse ->  AlfrescoSolrDataModel.getInstance()
+												.mapProperty(field, fieldUse, req)))
+						.filter(schemaFieldName -> schemaFieldName != null)
+						.map(schemaFieldName -> schemaFieldName.chars()
+								.mapToObj(c -> (char) c)
+								.map(c -> Character.isJavaIdentifierPart(c)? c : '?')
+								.map(Object::toString)
+								.collect(Collectors.joining()))
+						.collect(Collectors.toSet()));
+			}
+
+			params.add("fl", fieldListSet.stream().collect(Collectors.joining(",")));
+		}
+
+		// This is added for filtering the fields in the cached transformer.
+		params.set("originalFl", originalFieldList);
+		req.setParams(params);
+	}
+
 	@Override
 	public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
 			throws Exception {
 		readJsonIntoContent(req);
 
+		transformFieldList(req);
+
 		List<SearchComponent> components = getComponents();
 		ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
 		if (rb.requestInfo != null) {

search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml

@@ -1783,12 +1783,9 @@
      <transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" />
     -->
 
-  <transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
-  </transformer>
-
-  <transformer name="dv" class="org.alfresco.solr.transformer.DocValueDocTransformerFactory" >
-  </transformer>
+    <transformer name="cached" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" />
 
+    <transformer name="alfrescoMapper" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" />
 
   <!-- Legacy config for the admin interface -->
   <admin>

search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml

@@ -1785,11 +1785,11 @@
      <transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" />
     -->
     
-  <transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
-  </transformer>
+  <transformer name="cached" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" />
 
-  <transformer name="dv" class="org.alfresco.solr.transformer.DocValueDocTransformerFactory" >
-  </transformer>
+  <transformer name="alfrescoMapper" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" />
+
+  <transformer name="dv" class="org.alfresco.solr.transformer.DocValueDocTransformerFactory" />
 
 
   <!-- Legacy config for the admin interface -->

search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java

@@ -56,7 +56,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
 import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
 
 @SolrTestCaseJ4.SuppressSSL
-public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT
+public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistributedIT
 {
     public static final String ALFRESCO_JSON = "{\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}]}";
 
@@ -64,7 +64,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT
     public static void initData() throws Throwable
     {
         // FIXME: 1 shard??
-        initSolrServers(1, CachedDocTransformerIT.getSimpleClassName(), null);
+        initSolrServers(1, AlfrescoFieldMapperTransformerIT.getSimpleClassName(), null);
         populateAlfrescoData();
     }
 
@@ -183,6 +183,35 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT
         assertNotNull(docWithAllFields.get("DBID"));
     }
 
+    @Test
+    public void transformDocument_docTransformerGlobsFieldRequested_shouldReturnMatchedFields() throws Exception
+    {
+        putHandleDefaults();
+
+        QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm?title, *name, [cached]"));
+        assertNotNull(resp);
+        SolrDocumentList results = resp.getResults();
+        SolrDocument docWithAllFields = results.get(0);
+        assertEquals(4, docWithAllFields.size());
+        assertNotNull(docWithAllFields.get("cm_title"));
+        assertNotNull(docWithAllFields.get("cm:title"));
+        assertNotNull(docWithAllFields.get("cm_name"));
+    }
+
+    @Test
+    public void transformDocument_docTransformerIsUsedWithOtherTransformer_shouldExecuteBothTranformers() throws Exception
+    {
+        putHandleDefaults();
+
+        QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [explain], [cached]"));
+        assertNotNull(resp);
+        SolrDocumentList results = resp.getResults();
+        SolrDocument docWithAllFields = results.get(0);
+        assertEquals(2, docWithAllFields.size());
+        assertNotNull(docWithAllFields.get("cm_title"));
+        assertNotNull(docWithAllFields.get("[explain]"));
+    }
+
     private static void populateAlfrescoData() throws Exception
     {
         AclChangeSet aclChangeSet = getAclChangeSet(1);

search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrconfig.xml

@@ -553,7 +553,7 @@
     <searchComponent name="rewriteFacetCounts"     class="org.alfresco.solr.component.RewriteFacetCountsComponent" />
     <searchComponent name="setProcessedDenies"     class="org.alfresco.solr.component.SetProcessedDeniesComponent" />
 
-    <transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
+    <transformer name="cached" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" >
     </transformer>
 
 

search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml

@@ -559,7 +559,7 @@
     <searchComponent name="rewriteFacetCounts"     class="org.alfresco.solr.component.RewriteFacetCountsComponent" />
     <searchComponent name="setProcessedDenies"     class="org.alfresco.solr.component.SetProcessedDeniesComponent" />
 
-    <transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
+    <transformer name="cached" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" >
     </transformer>
 
 

search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml

@@ -558,7 +558,7 @@
     <searchComponent name="rewriteFacetCounts"     class="org.alfresco.solr.component.RewriteFacetCountsComponent" />
     <searchComponent name="setProcessedDenies"     class="org.alfresco.solr.component.SetProcessedDeniesComponent" />
 
-    <transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
+    <transformer name="cached" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" >
     </transformer>