From d5ce05f0214071a016e160aa45461de60591135c Mon Sep 17 00:00:00 2001 From: elia Date: Tue, 7 Jan 2020 18:13:28 +0100 Subject: [PATCH 1/9] [SEARCH-1689] added draft method for transforming field list --- .../component/AlfrescoSearchHandler.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index d2c21dc39..1011705b4 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -28,8 +28,10 @@ import java.io.Reader; import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.*; +import java.util.stream.Collectors; import org.alfresco.error.AlfrescoRuntimeException; +import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.query.AbstractQParser; import org.apache.lucene.document.Document; import org.apache.lucene.index.ExitableDirectoryReader; @@ -58,6 +60,7 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrQueryTimeoutImpl; +import org.apache.solr.search.SolrReturnFields; import org.apache.solr.search.facet.FacetModule; import org.apache.solr.util.RTimerTree; import org.apache.solr.util.SolrPluginUtils; @@ -287,11 +290,59 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements return shardHandler; } + private void transformFieldList(SolrQueryRequest req) + { + + StringBuilder flBuilder = new StringBuilder(); + SolrReturnFields solrReturnFields = new SolrReturnFields(req); + + boolean cacheTransformer = req.getParams().get("fl").contains("[cached]"); + + if (cacheTransformer){ + flBuilder.append("[cached] "); + } + + if (solrReturnFields.wantsAllFields()){ + if (!cacheTransformer){ + flBuilder.append("DBID"); + } else { + flBuilder.append("*"); + } + } else { + if (solrReturnFields.hasPatternMatching()){ + flBuilder.append("*"); + } else { + solrReturnFields.getLuceneFieldNames().forEach(field -> { + + + String schemaFieldName = AlfrescoSolrDataModel.getInstance() + .mapProperty(field, AlfrescoSolrDataModel.FieldUse.FTS, req); + + if (schemaFieldName != null){ + String trasformedSchemaField = schemaFieldName.chars() + .mapToObj(c -> (char) c) + .map(c -> Character.isJavaIdentifierPart(c)? c : '?') + .map(Object::toString) + .collect(Collectors.joining()); + + flBuilder.append(trasformedSchemaField); + } + }); + } + } + + ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); + params.set("fl", flBuilder.toString()); + req.setParams(params); + } + @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { readJsonIntoContent(req); +// transformFieldList(req); + List components = getComponents(); ResponseBuilder rb = new ResponseBuilder(req, rsp, components); if (rb.requestInfo != null) { From e35bf883887e496c245bb887768c29cc6c02692a Mon Sep 17 00:00:00 2001 From: elia Date: Wed, 8 Jan 2020 16:01:37 +0100 Subject: [PATCH 2/9] [SEARCH-1689] - added field list manipulation. - modified cached trasformer to use the input document --- .../transformer/CachedDocTransformer.java | 156 ++++++++---------- .../component/AlfrescoSearchHandler.java | 19 ++- 2 files changed, 85 insertions(+), 90 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java index 5d6e230b6..3f502c4e5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java @@ -32,10 +32,13 @@ import org.alfresco.solr.content.SolrContentStore; import org.apache.lucene.index.IndexableField; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.core.CoreContainer; +import org.apache.solr.response.DocsStreamer; import org.apache.solr.response.ResultContext; import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.SchemaField; +import org.codehaus.janino.Mod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,103 +73,86 @@ public class CachedDocTransformer extends DocTransformer @Override public void transform(SolrDocument doc, int docid, float score) throws IOException { - SolrInputDocument cachedDoc = null; - try + Collection fieldNames = new ArrayList<>(doc.getFieldNames()); + for (String fieldName : fieldNames) { - String id = getFieldValueString(doc, FIELD_SOLR4_ID); - TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(id); - CoreContainer coreContainer = context.getSearcher().getCore().getCoreContainer(); - AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler(); - SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers().get(context.getSearcher().getCore().getName()); - SolrContentStore solrContentStore = srv.getSolrContentStore(); - cachedDoc = solrContentStore.retrieveDocFromSolrContentStore(tenantAndDbId.tenant, tenantAndDbId.dbId); - } - catch(StringIndexOutOfBoundsException e) - { - // ignore invalid forms .... - } - - if(cachedDoc != null) - { - Collection fieldNames = cachedDoc.getFieldNames(); - for (String fieldName : fieldNames) - { - SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); - if(schemaField != null) + SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); + if(schemaField != null) + { + + boolean isTrasformedFieldName = isTrasformedField(fieldName); + if(schemaField.multiValued()) { + Collection values = doc.getFieldValues(fieldName); doc.removeFields(fieldName); - if(schemaField.multiValued()) + + if (!isTrasformedFieldName) { - int index = fieldName.lastIndexOf("@{"); - if(index == -1) - { - doc.addField(fieldName, cachedDoc.getFieldValues(fieldName)); - } - else - { - String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); - Collection values = cachedDoc.getFieldValues(fieldName); - - //Guard against null pointer in case data model field name does not match up with cachedDoc field name. - if(values != null) { - ArrayList newValues = new ArrayList(values.size()); - for (Object value : values) { - if (value instanceof String) { - String stringValue = (String) value; - int start = stringValue.lastIndexOf('\u0000'); - if (start == -1) { - newValues.add(stringValue); - } else { - newValues.add(stringValue.substring(start + 1)); - } - } else { - newValues.add(value); - } - - } - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, newValues); - } - } + doc.addField(fieldName, values); } else { - int index = fieldName.lastIndexOf("@{"); - if(index == -1) - { - doc.addField(fieldName, cachedDoc.getFieldValue(fieldName)); - } - else - { - String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); - alfrescoFieldName = alfrescoFieldName.contains(":") ? alfrescoFieldName.replace(":", "_") : alfrescoFieldName; - Object value = cachedDoc.getFieldValue(fieldName); - if(value instanceof String) - { - String stringValue = (String) value; - int start = stringValue.lastIndexOf('\u0000'); - if(start == -1) - { - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, stringValue); - } - else - { - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, stringValue.substring(start+1)); - } - } - else - { - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, value); + String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); + + //Guard against null pointer in case data model field name does not match up with cachedDoc field name. + if(values != null) { + ArrayList newValues = new ArrayList<>(values.size()); + for (Object value : values) { + newValues.add(getFieldValue(value)); } + doc.removeFields(alfrescoFieldName); + doc.addField(alfrescoFieldName, newValues); } } } - } - } + else + { + Object value = DocsStreamer.getValue(schemaField, (IndexableField) doc.getFieldValue(fieldName)); + doc.removeFields(fieldName); + if (!isTrasformedFieldName) + { + doc.removeFields(fieldName); + doc.addField(fieldName, value); + } + else + { + String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); + alfrescoFieldName = transformToUnderscoreNotation(alfrescoFieldName); + doc.removeFields(alfrescoFieldName); + doc.addField(alfrescoFieldName, getFieldValue(value)); + } + } + } + } + } + + private String transformToUnderscoreNotation(String value) + { + return value.contains(":") ? value.replace(":", "_") : value; + } + + private boolean isTrasformedField(String fieldName) + { + int index = fieldName.lastIndexOf("@{"); + return index != -1; + } + + private String removeLocale(String value) + { + int start = value.lastIndexOf('\u0000'); + if(start == -1){ + return value; + } else { + return value.substring(start + 1); + } + } + + private Object getFieldValue(Object value){ + if (value instanceof String) + return removeLocale((String) value ); + else + return value; } diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index 1011705b4..f0dc12950 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -294,23 +294,30 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements { StringBuilder flBuilder = new StringBuilder(); - SolrReturnFields solrReturnFields = new SolrReturnFields(req); + StringBuilder switchFlBuilder = new StringBuilder(); - boolean cacheTransformer = req.getParams().get("fl").contains("[cached]"); + SolrReturnFields solrReturnFields = new SolrReturnFields(req); + String oldFieldList = req.getParams().get("fl"); + + boolean cacheTransformer = oldFieldList != null && oldFieldList.contains("[cached]"); if (cacheTransformer){ flBuilder.append("[cached] "); + switchFlBuilder.append("[cached] "); } if (solrReturnFields.wantsAllFields()){ if (!cacheTransformer){ - flBuilder.append("DBID"); + flBuilder.append("id, DBID, _version_"); + switchFlBuilder.append("id, DBID, _version_"); } else { flBuilder.append("*"); + switchFlBuilder.append("*"); } } else { if (solrReturnFields.hasPatternMatching()){ flBuilder.append("*"); + switchFlBuilder.append("*"); } else { solrReturnFields.getLuceneFieldNames().forEach(field -> { @@ -325,7 +332,8 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements .map(Object::toString) .collect(Collectors.joining()); - flBuilder.append(trasformedSchemaField); + flBuilder.append( trasformedSchemaField + ", "); + flBuilder.append( field + ", "); } }); } @@ -333,6 +341,7 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); params.set("fl", flBuilder.toString()); + params.set("switchFl", switchFlBuilder.toString()); req.setParams(params); } @@ -341,7 +350,7 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements throws Exception { readJsonIntoContent(req); -// transformFieldList(req); + transformFieldList(req); List components = getComponents(); ResponseBuilder rb = new ResponseBuilder(req, rsp, components); From 9711102e80291bfc057a91949257fb5ff856140a Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 9 Jan 2020 17:20:50 +0100 Subject: [PATCH 3/9] [SEARCH-1689] Handled globs in return fields. --- .../transformer/CachedDocTransformer.java | 73 +++++++++------ .../component/AlfrescoSearchHandler.java | 92 +++++++++++-------- 2 files changed, 101 insertions(+), 64 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java index 3f502c4e5..9c88c00d1 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java @@ -38,6 +38,7 @@ import org.apache.solr.response.DocsStreamer; import org.apache.solr.response.ResultContext; import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SolrReturnFields; import org.codehaus.janino.Mod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,7 @@ public class CachedDocTransformer extends DocTransformer protected final static Logger log = LoggerFactory.getLogger(CachedDocTransformer.class); private ResultContext context; + private SolrReturnFields solrReturnFields; /* (non-Javadoc) * @see org.apache.solr.response.transform.DocTransformer#getName() @@ -74,59 +76,74 @@ public class CachedDocTransformer extends DocTransformer public void transform(SolrDocument doc, int docid, float score) throws IOException { Collection fieldNames = new ArrayList<>(doc.getFieldNames()); + solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest()); for (String fieldName : fieldNames) { SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); + if(schemaField != null) { - boolean isTrasformedFieldName = isTrasformedField(fieldName); - if(schemaField.multiValued()) + String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); + if (isRequestedField(alfrescoFieldName)) { - Collection values = doc.getFieldValues(fieldName); - doc.removeFields(fieldName); - - if (!isTrasformedFieldName) + if(schemaField.multiValued()) { - doc.addField(fieldName, values); + Collection values = doc.getFieldValues(fieldName); + doc.removeFields(fieldName); + + if (!isTrasformedFieldName) + { + doc.addField(fieldName, values); + } + else + { + //Guard against null pointer in case data model field name does not match up with cachedDoc field name. + if(values != null) { + ArrayList newValues = new ArrayList<>(values.size()); + for (Object value : values) { + newValues.add(getFieldValue(value)); + } + doc.removeFields(alfrescoFieldName); + doc.addField(alfrescoFieldName, newValues); + } + } } else { - String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); + Object value = DocsStreamer.getValue(schemaField, (IndexableField) doc.getFieldValue(fieldName)); + doc.removeFields(fieldName); - //Guard against null pointer in case data model field name does not match up with cachedDoc field name. - if(values != null) { - ArrayList newValues = new ArrayList<>(values.size()); - for (Object value : values) { - newValues.add(getFieldValue(value)); - } + if (!isTrasformedFieldName) + { + doc.removeFields(fieldName); + doc.addField(fieldName, value); + } + else + { + alfrescoFieldName = transformToUnderscoreNotation(alfrescoFieldName); doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, newValues); + doc.addField(alfrescoFieldName, getFieldValue(value)); } } } else { - Object value = DocsStreamer.getValue(schemaField, (IndexableField) doc.getFieldValue(fieldName)); - doc.removeFields(fieldName); - - if (!isTrasformedFieldName) + if (!fieldName.equals("id")) { - doc.removeFields(fieldName); - doc.addField(fieldName, value); - } - else - { - String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); - alfrescoFieldName = transformToUnderscoreNotation(alfrescoFieldName); - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, getFieldValue(value)); + doc.remove(fieldName); + doc.remove(alfrescoFieldName); } } } } } + private boolean isRequestedField(String fieldName) + { + return solrReturnFields.wantsField(fieldName.replace(":", "_")); + } + private String transformToUnderscoreNotation(String value) { return value.contains(":") ? value.replace(":", "_") : value; diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index f0dc12950..248bf9f5c 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -18,6 +18,10 @@ */ package org.apache.solr.handler.component; +import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FACET; +import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FTS; +import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.ID; +import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.SORT; import static org.apache.solr.common.params.CommonParams.PATH; import java.io.BufferedReader; @@ -29,10 +33,12 @@ import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.query.AbstractQParser; +import org.apache.cxf.transport.http.auth.HttpAuthHeader; import org.apache.lucene.document.Document; import org.apache.lucene.index.ExitableDirectoryReader; import org.apache.lucene.index.IndexableField; @@ -293,55 +299,69 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements private void transformFieldList(SolrQueryRequest req) { - StringBuilder flBuilder = new StringBuilder(); - StringBuilder switchFlBuilder = new StringBuilder(); + Set fieldListSet = new HashSet<>(); + Set allowedNonCachedFields = Set.of("id","DBID", "_version_", "score"); SolrReturnFields solrReturnFields = new SolrReturnFields(req); - String oldFieldList = req.getParams().get("fl"); + String originalFieldList = req.getParams().get("fl"); - boolean cacheTransformer = oldFieldList != null && oldFieldList.contains("[cached]"); + boolean cacheTransformer = originalFieldList != null && originalFieldList.contains("[cached]"); - if (cacheTransformer){ - flBuilder.append("[cached] "); - switchFlBuilder.append("[cached] "); + if (cacheTransformer) + { + fieldListSet.add("[cached]"); } - if (solrReturnFields.wantsAllFields()){ - if (!cacheTransformer){ - flBuilder.append("id, DBID, _version_"); - switchFlBuilder.append("id, DBID, _version_"); - } else { - flBuilder.append("*"); - switchFlBuilder.append("*"); + if (solrReturnFields.wantsAllFields()) + { + if (!cacheTransformer) + { + fieldListSet.addAll(Set.of("id","DBID", "_version_")); } - } else { - if (solrReturnFields.hasPatternMatching()){ - flBuilder.append("*"); - switchFlBuilder.append("*"); - } else { - solrReturnFields.getLuceneFieldNames().forEach(field -> { + else + { + fieldListSet.add("*"); + } + } + else + { + if (solrReturnFields.hasPatternMatching()) + { + fieldListSet.add("*"); + } + else + { + if (!cacheTransformer) + { + fieldListSet.addAll(solrReturnFields.getLuceneFieldNames() + .stream() + .filter(field -> allowedNonCachedFields.contains(field)) + .collect(Collectors.toSet())); + } + else + { - String schemaFieldName = AlfrescoSolrDataModel.getInstance() - .mapProperty(field, AlfrescoSolrDataModel.FieldUse.FTS, req); - - if (schemaFieldName != null){ - String trasformedSchemaField = schemaFieldName.chars() - .mapToObj(c -> (char) c) - .map(c -> Character.isJavaIdentifierPart(c)? c : '?') - .map(Object::toString) - .collect(Collectors.joining()); - - flBuilder.append( trasformedSchemaField + ", "); - flBuilder.append( field + ", "); - } - }); + List fieldUsed = List.of(FTS, FACET, ID, SORT); + fieldListSet.addAll(solrReturnFields.getLuceneFieldNames().stream() + .flatMap(field -> + Stream.concat(Stream.of(field), fieldUsed.stream() + .map( fieldUse -> AlfrescoSolrDataModel.getInstance() + .mapProperty(field, fieldUse, req)))) + .filter(schemaFieldName -> schemaFieldName != null) + .map(schemaFieldName -> schemaFieldName.chars() + .mapToObj(c -> (char) c) + .map(c -> Character.isJavaIdentifierPart(c)? c : '?') + .map(Object::toString) + .collect(Collectors.joining())) + .collect(Collectors.toSet())); + } } } ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); - params.set("fl", flBuilder.toString()); - params.set("switchFl", switchFlBuilder.toString()); + params.set("fl", fieldListSet.stream().collect(Collectors.joining(","))); + params.set("originalFl", originalFieldList); req.setParams(params); } From 632f147024083f1969774824d84fa9974ed71481 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 10 Jan 2020 15:44:21 +0100 Subject: [PATCH 4/9] [SEARCH-1689] - refactoring - handled shard queries - Added integration test for glob queries --- .../transformer/CachedDocTransformer.java | 124 ++++-------------- .../component/AlfrescoSearchHandler.java | 64 ++++----- .../transformer/CachedDocTransformerIT.java | 15 +++ 3 files changed, 67 insertions(+), 136 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java index 9c88c00d1..b75c78e71 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2005-2014 Alfresco Software Limited. + * Copyright (C) 2005-2020 Alfresco Software Limited. * * This file is part of Alfresco * @@ -18,33 +18,23 @@ */ package org.alfresco.solr.transformer; -import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; - -import org.alfresco.solr.AlfrescoCoreAdminHandler; import org.alfresco.solr.AlfrescoSolrDataModel; -import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId; -import org.alfresco.solr.SolrInformationServer; -import org.alfresco.solr.content.SolrContentStore; import org.apache.lucene.index.IndexableField; import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.core.CoreContainer; import org.apache.solr.response.DocsStreamer; import org.apache.solr.response.ResultContext; import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.SolrReturnFields; -import org.codehaus.janino.Mod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; + /** - * @author Andy + * @author Andy, Elia * */ public class CachedDocTransformer extends DocTransformer @@ -73,67 +63,39 @@ public class CachedDocTransformer extends DocTransformer * @see org.apache.solr.response.transform.DocTransformer#transform(org.apache.solr.common.SolrDocument, int) */ @Override - public void transform(SolrDocument doc, int docid, float score) throws IOException - { + public void transform(SolrDocument doc, int docid, float score) { Collection fieldNames = new ArrayList<>(doc.getFieldNames()); solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest()); + + for (String fieldName : fieldNames) { SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); - if(schemaField != null) { - boolean isTrasformedFieldName = isTrasformedField(fieldName); String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); - if (isRequestedField(alfrescoFieldName)) + if (isRequestedField(alfrescoFieldName) || alfrescoFieldName.equals("id")) { - if(schemaField.multiValued()) + Object value = doc.getFieldValue(fieldName); + doc.removeFields(fieldName); + if (schemaField.multiValued()) { - Collection values = doc.getFieldValues(fieldName); - doc.removeFields(fieldName); - - if (!isTrasformedFieldName) - { - doc.addField(fieldName, values); - } - else - { - //Guard against null pointer in case data model field name does not match up with cachedDoc field name. - if(values != null) { - ArrayList newValues = new ArrayList<>(values.size()); - for (Object value : values) { - newValues.add(getFieldValue(value)); - } - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, newValues); - } - } + Object collectionValue = ((Collection) value).stream() + .map(elem -> getFieldValue(schemaField, elem)) + .collect(Collectors.toSet()); + doc.setField(alfrescoFieldName, collectionValue); } else { - Object value = DocsStreamer.getValue(schemaField, (IndexableField) doc.getFieldValue(fieldName)); - doc.removeFields(fieldName); - - if (!isTrasformedFieldName) - { - doc.removeFields(fieldName); - doc.addField(fieldName, value); - } - else - { - alfrescoFieldName = transformToUnderscoreNotation(alfrescoFieldName); - doc.removeFields(alfrescoFieldName); - doc.addField(alfrescoFieldName, getFieldValue(value)); - } + doc.setField(transformToUnderscoreNotation(alfrescoFieldName), + getFieldValue(schemaField, value)); } + } else { - if (!fieldName.equals("id")) - { - doc.remove(fieldName); - doc.remove(alfrescoFieldName); - } + doc.removeFields(alfrescoFieldName); + doc.removeFields(fieldName); } } } @@ -144,17 +106,11 @@ public class CachedDocTransformer extends DocTransformer return solrReturnFields.wantsField(fieldName.replace(":", "_")); } - private String transformToUnderscoreNotation(String value) + private static String transformToUnderscoreNotation(String value) { return value.contains(":") ? value.replace(":", "_") : value; } - private boolean isTrasformedField(String fieldName) - { - int index = fieldName.lastIndexOf("@{"); - return index != -1; - } - private String removeLocale(String value) { int start = value.lastIndexOf('\u0000'); @@ -165,36 +121,8 @@ public class CachedDocTransformer extends DocTransformer } } - private Object getFieldValue(Object value){ - if (value instanceof String) - return removeLocale((String) value ); - else - return value; - } - - - private String getFieldValueString(SolrDocument doc, String fieldName) - { - Object o = doc.getFieldValue(fieldName); - if(o != null) - { - if(o instanceof IndexableField) - { - IndexableField field = (IndexableField)o; - return field.stringValue(); - } - else if(o instanceof String) - { - return (String)o; - } - else - { - return o.toString(); - } - } - else - { - return null; - } + private Object getFieldValue(SchemaField schemaField, Object value){ + Object indexedValue = DocsStreamer.getValue(schemaField, (IndexableField) value); + return indexedValue instanceof String? removeLocale((String) indexedValue) : indexedValue; } } diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index 248bf9f5c..d08e83bc1 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -298,6 +298,8 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements private void transformFieldList(SolrQueryRequest req) { + if (req.getParams().get("originalFl") != null) + return; Set fieldListSet = new HashSet<>(); @@ -306,61 +308,47 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements String originalFieldList = req.getParams().get("fl"); boolean cacheTransformer = originalFieldList != null && originalFieldList.contains("[cached]"); - - if (cacheTransformer) - { - fieldListSet.add("[cached]"); - } - - if (solrReturnFields.wantsAllFields()) - { - if (!cacheTransformer) + ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); + if (!cacheTransformer){ + if (solrReturnFields.wantsAllFields()) { fieldListSet.addAll(Set.of("id","DBID", "_version_")); } else { - fieldListSet.add("*"); + fieldListSet.addAll(solrReturnFields.getLuceneFieldNames() + .stream() + .filter(field -> allowedNonCachedFields.contains(field)) + .collect(Collectors.toSet())); } + params.set("fl", fieldListSet.stream().collect(Collectors.joining(","))); } else { - if (solrReturnFields.hasPatternMatching()) + if (solrReturnFields.wantsAllFields() || solrReturnFields.hasPatternMatching()) { fieldListSet.add("*"); } else { - - if (!cacheTransformer) - { - fieldListSet.addAll(solrReturnFields.getLuceneFieldNames() - .stream() - .filter(field -> allowedNonCachedFields.contains(field)) - .collect(Collectors.toSet())); - } - else - { - - List fieldUsed = List.of(FTS, FACET, ID, SORT); - fieldListSet.addAll(solrReturnFields.getLuceneFieldNames().stream() - .flatMap(field -> - Stream.concat(Stream.of(field), fieldUsed.stream() - .map( fieldUse -> AlfrescoSolrDataModel.getInstance() - .mapProperty(field, fieldUse, req)))) - .filter(schemaFieldName -> schemaFieldName != null) - .map(schemaFieldName -> schemaFieldName.chars() - .mapToObj(c -> (char) c) - .map(c -> Character.isJavaIdentifierPart(c)? c : '?') - .map(Object::toString) - .collect(Collectors.joining())) - .collect(Collectors.toSet())); - } + List fieldUsed = List.of(FTS, FACET, ID, SORT); + fieldListSet.addAll(solrReturnFields.getLuceneFieldNames().stream() + .flatMap(field -> + fieldUsed.stream() + .map( fieldUse -> AlfrescoSolrDataModel.getInstance() + .mapProperty(field, fieldUse, req))) + .filter(schemaFieldName -> schemaFieldName != null) + .map(schemaFieldName -> schemaFieldName.chars() + .mapToObj(c -> (char) c) + .map(c -> Character.isJavaIdentifierPart(c)? c : '?') + .map(Object::toString) + .collect(Collectors.joining())) + .collect(Collectors.toSet())); } + + params.add("fl", fieldListSet.stream().collect(Collectors.joining(","))); } - ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); - params.set("fl", fieldListSet.stream().collect(Collectors.joining(","))); params.set("originalFl", originalFieldList); req.setParams(params); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java index f5d60d098..a07be8b10 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java @@ -183,6 +183,21 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT assertNotNull(docWithAllFields.get("DBID")); } + @Test + public void transformDocument_docTransformerGlobsFieldRequested_shouldReturnCorrespondingFields() throws Exception + { + putHandleDefaults(); + + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm?title, *name, [cached]")); + assertNotNull(resp); + SolrDocumentList results = resp.getResults(); + SolrDocument docWithAllFields = results.get(0); + assertEquals(4, docWithAllFields.size()); + assertNotNull(docWithAllFields.get("cm_title")); + assertNotNull(docWithAllFields.get("cm:title")); + assertNotNull(docWithAllFields.get("cm_name")); + } + private static void populateAlfrescoData() throws Exception { AclChangeSet aclChangeSet = getAclChangeSet(1); From f33e696b3e47443ae72484140eada96dfa542bb5 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Fri, 10 Jan 2020 16:00:55 +0100 Subject: [PATCH 5/9] [SEARCH-1689] Added integration test for testing two trasformers. --- .../handler/component/AlfrescoSearchHandler.java | 9 +++++++-- .../solr/transformer/CachedDocTransformerIT.java | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index d08e83bc1..8423a8136 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -303,16 +303,20 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements Set fieldListSet = new HashSet<>(); - Set allowedNonCachedFields = Set.of("id","DBID", "_version_", "score"); + Set allowedNonCachedFields = Set.of("id","DBID", "_version_"); SolrReturnFields solrReturnFields = new SolrReturnFields(req); String originalFieldList = req.getParams().get("fl"); boolean cacheTransformer = originalFieldList != null && originalFieldList.contains("[cached]"); ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); + + + // In case cache transformer is no set, we need to modify the field list in order return + // only id, DBID and _version_ fields if (!cacheTransformer){ if (solrReturnFields.wantsAllFields()) { - fieldListSet.addAll(Set.of("id","DBID", "_version_")); + fieldListSet.addAll(allowedNonCachedFields); } else { @@ -321,6 +325,7 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements .filter(field -> allowedNonCachedFields.contains(field)) .collect(Collectors.toSet())); } + params.set("fl", fieldListSet.stream().collect(Collectors.joining(","))); } else diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java index a07be8b10..28ba348c3 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java @@ -184,7 +184,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT } @Test - public void transformDocument_docTransformerGlobsFieldRequested_shouldReturnCorrespondingFields() throws Exception + public void transformDocument_docTransformerGlobsFieldRequested_shouldReturnMatchedFields() throws Exception { putHandleDefaults(); @@ -198,6 +198,20 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT assertNotNull(docWithAllFields.get("cm_name")); } + @Test + public void transformDocument_docTransformerIsUsedWithOtherTransformer_shouldExecuteBothTranformers() throws Exception + { + putHandleDefaults(); + + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [cached], [explain]")); + assertNotNull(resp); + SolrDocumentList results = resp.getResults(); + SolrDocument docWithAllFields = results.get(0); + assertEquals(2, docWithAllFields.size()); + assertNotNull(docWithAllFields.get("cm_title")); + assertNotNull(docWithAllFields.get("[explain]")); + } + private static void populateAlfrescoData() throws Exception { AclChangeSet aclChangeSet = getAclChangeSet(1); From baf8167a02c55fb33ad6a8bc4b0c352fcea699e7 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Mon, 13 Jan 2020 16:27:31 +0100 Subject: [PATCH 6/9] [SEARCH-1689] Added score to allowed non cached fields --- .../solr/handler/component/AlfrescoSearchHandler.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index 8423a8136..618997318 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -303,7 +303,10 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements Set fieldListSet = new HashSet<>(); - Set allowedNonCachedFields = Set.of("id","DBID", "_version_"); + Set defaultNonCachedFields = Set.of("id","DBID", "_version_"); + Set allowedNonCachedFields = new HashSet<>(defaultNonCachedFields); + allowedNonCachedFields.add("score"); + SolrReturnFields solrReturnFields = new SolrReturnFields(req); String originalFieldList = req.getParams().get("fl"); @@ -316,7 +319,7 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements if (!cacheTransformer){ if (solrReturnFields.wantsAllFields()) { - fieldListSet.addAll(allowedNonCachedFields); + fieldListSet.addAll(defaultNonCachedFields); } else { From 1504ad392acdc2298f0622f25c2b0ebde0545ccb Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 14 Jan 2020 14:10:11 +0100 Subject: [PATCH 7/9] [SEARCH-1689] added comments --- .../alfresco/solr/transformer/CachedDocTransformer.java | 3 +-- .../solr/handler/component/AlfrescoSearchHandler.java | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java index b75c78e71..045a3e32d 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java @@ -67,7 +67,6 @@ public class CachedDocTransformer extends DocTransformer Collection fieldNames = new ArrayList<>(doc.getFieldNames()); solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest()); - for (String fieldName : fieldNames) { SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); @@ -103,7 +102,7 @@ public class CachedDocTransformer extends DocTransformer private boolean isRequestedField(String fieldName) { - return solrReturnFields.wantsField(fieldName.replace(":", "_")); + return solrReturnFields.wantsField(transformToUnderscoreNotation(fieldName)); } private static String transformToUnderscoreNotation(String value) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index 618997318..5247ca8ee 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -296,6 +296,14 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements return shardHandler; } + + /** + * Transform the fieldlist depending on the use of cached transformer: + * [cached] -> add to the field list the translations of the fiels to the internal schema notation + * otherwise -> modify the field list in order to contains a subset of the following fields: + * id, DBID, _version_ and score + * @param req + */ private void transformFieldList(SolrQueryRequest req) { if (req.getParams().get("originalFl") != null) @@ -357,6 +365,7 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements params.add("fl", fieldListSet.stream().collect(Collectors.joining(","))); } + // This is added for filtering the fields in the cached transformer. params.set("originalFl", originalFieldList); req.setParams(params); } From 6623460765372f197d6df1b89d23c33248508886 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 14 Jan 2020 15:42:17 +0100 Subject: [PATCH 8/9] [SEARCH-1689] Modified cached transformer name. Code refactoring --- ...va => AlfrescoFieldMapperTransformer.java} | 254 +++++++++--------- ...lfrescoFieldMapperTransformerFactory.java} | 84 +++--- .../component/AlfrescoSearchHandler.java | 2 +- .../templates/noRerank/conf/solrconfig.xml | 7 +- .../templates/rerank/conf/solrconfig.xml | 8 +- ... => AlfrescoFieldMapperTransformerIT.java} | 6 +- .../collection1/conf/solrconfig.xml | 2 +- .../test-files/master/conf/solrconfig.xml | 2 +- .../test-files/slave/conf/solrconfig.xml | 2 +- 9 files changed, 182 insertions(+), 185 deletions(-) rename search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/{CachedDocTransformer.java => AlfrescoFieldMapperTransformer.java} (94%) rename search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/{CachedDocTransformerFactory.java => AlfrescoFieldMapperTransformerFactory.java} (89%) rename search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/{CachedDocTransformerIT.java => AlfrescoFieldMapperTransformerIT.java} (98%) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformer.java similarity index 94% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformer.java index 045a3e32d..5977dc4db 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformer.java @@ -1,127 +1,127 @@ -/* - * Copyright (C) 2005-2020 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.transformer; - -import org.alfresco.solr.AlfrescoSolrDataModel; -import org.apache.lucene.index.IndexableField; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.response.DocsStreamer; -import org.apache.solr.response.ResultContext; -import org.apache.solr.response.transform.DocTransformer; -import org.apache.solr.schema.SchemaField; -import org.apache.solr.search.SolrReturnFields; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.stream.Collectors; - -/** - * @author Andy, Elia - * - */ -public class CachedDocTransformer extends DocTransformer -{ - protected final static Logger log = LoggerFactory.getLogger(CachedDocTransformer.class); - - private ResultContext context; - private SolrReturnFields solrReturnFields; - - /* (non-Javadoc) - * @see org.apache.solr.response.transform.DocTransformer#getName() - */ - @Override - public String getName() - { - return "Alfresco cached document transformer"; - } - - - public void setContext( ResultContext context ) - { - this.context = context; - } - - /* (non-Javadoc) - * @see org.apache.solr.response.transform.DocTransformer#transform(org.apache.solr.common.SolrDocument, int) - */ - @Override - public void transform(SolrDocument doc, int docid, float score) { - Collection fieldNames = new ArrayList<>(doc.getFieldNames()); - solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest()); - - for (String fieldName : fieldNames) - { - SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); - if(schemaField != null) - { - String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); - if (isRequestedField(alfrescoFieldName) || alfrescoFieldName.equals("id")) - { - Object value = doc.getFieldValue(fieldName); - doc.removeFields(fieldName); - if (schemaField.multiValued()) - { - Object collectionValue = ((Collection) value).stream() - .map(elem -> getFieldValue(schemaField, elem)) - .collect(Collectors.toSet()); - doc.setField(alfrescoFieldName, collectionValue); - } - else - { - doc.setField(transformToUnderscoreNotation(alfrescoFieldName), - getFieldValue(schemaField, value)); - } - - } - else - { - doc.removeFields(alfrescoFieldName); - doc.removeFields(fieldName); - } - } - } - } - - private boolean isRequestedField(String fieldName) - { - return solrReturnFields.wantsField(transformToUnderscoreNotation(fieldName)); - } - - private static String transformToUnderscoreNotation(String value) - { - return value.contains(":") ? value.replace(":", "_") : value; - } - - private String removeLocale(String value) - { - int start = value.lastIndexOf('\u0000'); - if(start == -1){ - return value; - } else { - return value.substring(start + 1); - } - } - - private Object getFieldValue(SchemaField schemaField, Object value){ - Object indexedValue = DocsStreamer.getValue(schemaField, (IndexableField) value); - return indexedValue instanceof String? removeLocale((String) indexedValue) : indexedValue; - } -} +/* + * Copyright (C) 2005-2020 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.transformer; + +import org.alfresco.solr.AlfrescoSolrDataModel; +import org.apache.lucene.index.IndexableField; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.response.DocsStreamer; +import org.apache.solr.response.ResultContext; +import org.apache.solr.response.transform.DocTransformer; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SolrReturnFields; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; + +/** + * @author Andy, Elia + * + */ +public class AlfrescoFieldMapperTransformer extends DocTransformer +{ + protected final static Logger log = LoggerFactory.getLogger(AlfrescoFieldMapperTransformer.class); + + private ResultContext context; + private SolrReturnFields solrReturnFields; + + /* (non-Javadoc) + * @see org.apache.solr.response.transform.DocTransformer#getName() + */ + @Override + public String getName() + { + return "alfrescoMapper"; + } + + + public void setContext( ResultContext context ) + { + this.context = context; + } + + /* (non-Javadoc) + * @see org.apache.solr.response.transform.DocTransformer#transform(org.apache.solr.common.SolrDocument, int) + */ + @Override + public void transform(SolrDocument doc, int docid, float score) { + Collection fieldNames = new ArrayList<>(doc.getFieldNames()); + solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest()); + + for (String fieldName : fieldNames) + { + SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName); + if(schemaField != null) + { + String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName); + if (isRequestedField(alfrescoFieldName) || alfrescoFieldName.equals("id")) + { + Object value = doc.getFieldValue(fieldName); + doc.removeFields(fieldName); + if (schemaField.multiValued()) + { + Object collectionValue = ((Collection) value).stream() + .map(elem -> getFieldValue(schemaField, elem)) + .collect(Collectors.toSet()); + doc.setField(alfrescoFieldName, collectionValue); + } + else + { + doc.setField(transformToUnderscoreNotation(alfrescoFieldName), + getFieldValue(schemaField, value)); + } + + } + else + { + doc.removeFields(alfrescoFieldName); + doc.removeFields(fieldName); + } + } + } + } + + private boolean isRequestedField(String fieldName) + { + return solrReturnFields.wantsField(transformToUnderscoreNotation(fieldName)); + } + + private static String transformToUnderscoreNotation(String value) + { + return value.contains(":") ? value.replace(":", "_") : value; + } + + private String removeLocale(String value) + { + int start = value.lastIndexOf('\u0000'); + if(start == -1){ + return value; + } else { + return value.substring(start + 1); + } + } + + private Object getFieldValue(SchemaField schemaField, Object value){ + Object indexedValue = DocsStreamer.getValue(schemaField, (IndexableField) value); + return indexedValue instanceof String? removeLocale((String) indexedValue) : indexedValue; + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformerFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerFactory.java similarity index 89% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformerFactory.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerFactory.java index f11954b97..fad9c154a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformerFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerFactory.java @@ -1,42 +1,42 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.transformer; - -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.transform.DocTransformer; -import org.apache.solr.response.transform.TransformerFactory; - -/** - * @author Andy - * - */ -public class CachedDocTransformerFactory extends TransformerFactory -{ - - /* (non-Javadoc) - * @see org.apache.solr.response.transform.TransformerFactory#create(java.lang.String, org.apache.solr.common.params.SolrParams, org.apache.solr.request.SolrQueryRequest) - */ - @Override - public DocTransformer create(String field, SolrParams params, SolrQueryRequest req) - { - return new CachedDocTransformer(); - } - -} +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.transformer; + +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.transform.DocTransformer; +import org.apache.solr.response.transform.TransformerFactory; + +/** + * @author Andy + * + */ +public class AlfrescoFieldMapperTransformerFactory extends TransformerFactory +{ + + /* (non-Javadoc) + * @see org.apache.solr.response.transform.TransformerFactory#create(java.lang.String, org.apache.solr.common.params.SolrParams, org.apache.solr.request.SolrQueryRequest) + */ + @Override + public DocTransformer create(String field, SolrParams params, SolrQueryRequest req) + { + return new AlfrescoFieldMapperTransformer(); + } + +} diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index 5247ca8ee..8ffcc56fd 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -318,7 +318,7 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements SolrReturnFields solrReturnFields = new SolrReturnFields(req); String originalFieldList = req.getParams().get("fl"); - boolean cacheTransformer = originalFieldList != null && originalFieldList.contains("[cached]"); + boolean cacheTransformer = originalFieldList != null && solrReturnFields.getTransformer().getName().contains("alfrescoMapper"); ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml index 331a4de85..ef42a2ca8 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml @@ -1782,13 +1782,10 @@ EditorialMarkerFactory will do exactly that: --> - - - - - + + diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml index 60a2dfdae..3d4477288 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrconfig.xml @@ -1785,11 +1785,11 @@ --> - - + - - + + + diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java similarity index 98% rename from search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java rename to search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java index 28ba348c3..24c3ddc29 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/CachedDocTransformerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java @@ -56,7 +56,7 @@ import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; @SolrTestCaseJ4.SuppressSSL -public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT +public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistributedIT { public static final String ALFRESCO_JSON = "{\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}]}"; @@ -64,7 +64,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT public static void initData() throws Throwable { // FIXME: 1 shard?? - initSolrServers(1, CachedDocTransformerIT.getSimpleClassName(), null); + initSolrServers(1, AlfrescoFieldMapperTransformerIT.getSimpleClassName(), null); populateAlfrescoData(); } @@ -203,7 +203,7 @@ public class CachedDocTransformerIT extends AbstractAlfrescoDistributedIT { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [cached], [explain]")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [explain], [cached]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); diff --git a/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrconfig.xml index 55917d17c..a84413271 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/collection1/conf/solrconfig.xml @@ -553,7 +553,7 @@ - + diff --git a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml index c05053d6e..a62f2cb9a 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/master/conf/solrconfig.xml @@ -559,7 +559,7 @@ - + diff --git a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml index 8d92bca34..edb360b9f 100644 --- a/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml +++ b/search-services/alfresco-search/src/test/resources/test-files/slave/conf/solrconfig.xml @@ -558,7 +558,7 @@ - + From 53e35e7ada222a12bdfbaa03cb789ef122f601d4 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Tue, 14 Jan 2020 22:51:47 +0100 Subject: [PATCH 9/9] [SEARCH-1689] Fixed NPE in field list transform --- .../solr/handler/component/AlfrescoSearchHandler.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java index 8ffcc56fd..b544cd1de 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java @@ -18,6 +18,8 @@ */ package org.apache.solr.handler.component; +import static java.util.Optional.of; +import static java.util.Optional.ofNullable; import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FACET; import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FTS; import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.ID; @@ -318,7 +320,11 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements SolrReturnFields solrReturnFields = new SolrReturnFields(req); String originalFieldList = req.getParams().get("fl"); - boolean cacheTransformer = originalFieldList != null && solrReturnFields.getTransformer().getName().contains("alfrescoMapper"); + boolean cacheTransformer = ofNullable(solrReturnFields.getTransformer()) + .map(t -> t.getName()) + .map(name -> name.contains("alfrescoMapper")) + .orElse(false); + ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());