From c555f6d400eeb0d9e192cac4cf0113e5c6ffa5e5 Mon Sep 17 00:00:00 2001 From: Elia Porciani Date: Fri, 11 Sep 2020 14:41:16 +0200 Subject: [PATCH] Fix/search 2400 to 2.0.x (#937) * [SEARCH-2400] restored shared.properties as in 1.4.3 (cherry picked from commit 45fd7b6c83b7f96925a163853ee22cc09c52ba44) * [SEARCH-2400] changed music model (cherry picked from commit f7bbac444de7c2a7ea9bf18e72ea1b43153344c0) * [SEARCH-2400] removed conf directory from solrhome in distributed tests (cherry picked from commit 52c94cef08d1d5646b37074ba42b1ab6ee2ab1c7) * [SEARCH-2400] use different shared.properties for integration tests (cherry picked from commit 61dd7a5df731b2a4e88f6302d89b738b35c6f5f4) * [SEARCH-2400] restored music model (cherry picked from commit b5894e6ddb5a9acbf7a0dd8e4d6c8f0014ed5d69) * [SEARCh-2400] restored test to be the same as in 1.4.3 branch (cherry picked from commit d5c6ced530b4edaae00a8edbde55b9fdffcc6839) * [SEARCH-2400] activate cross-locale field for all the non-tokenized properties (cherry picked from commit e04d486f623407dc3c763f637d3bdd90937364c2) * [SEARCH-2400] fixed test on dynamic fields (cherry picked from commit b336d8ff02f7b7e29202160ef1b8081a37d86c53) * [SEARCH-2400] fixed test shared.properties on insightEngine (cherry picked from commit 5a0738eed5ec1684d2100a15992294963e50431c) * [SEARCH-2400] removed check for property with no cross-locale fields in highlighter (cherry picked from commit a05149503e30556e299e63122acbb5e43dc62ff6) --- .../search/SearchAFTSInFieldTest.java | 4 +- .../generate-solr-config.py | 4 +- search-services/alfresco-search/pom.xml | 18 + .../component/AlfrescoSolrHighlighter.java | 9 +- .../solr/instance/conf/shared.properties | 14 +- .../rerank/conf/generated_copy_fields.xml | 652 +++++++++--------- .../alfresco/solr/DynamicCopyFieldsIT.java | 12 +- .../test-files/conf/shared.properties | 36 + 8 files changed, 411 insertions(+), 338 deletions(-) create mode 100644 search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java index 6fc22c2f3..92130b0d3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java @@ -137,13 +137,13 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest boolean fileFound = isContentInSearchResults(query, file2.getName(), true); Assert.assertTrue(fileFound, "File Not found for query: " + query); - testSearchQuery(query, 2, SearchLanguage.AFTS); + testSearchQuery(query, 1, SearchLanguage.AFTS); query = fieldName + ":'" + file2.getName() + "\'"; fileFound = isContentInSearchResults(query, file2.getName(), true); Assert.assertTrue(fileFound, "File Not found for query: " + query); - testSearchQuery(query, 2, SearchLanguage.AFTS); + testSearchQuery(query, 1, SearchLanguage.AFTS); }); } diff --git a/search-services/alfresco-search/generator-solr-config/generate-solr-config.py b/search-services/alfresco-search/generator-solr-config/generate-solr-config.py index 927385845..4c4ffffb4 100644 --- a/search-services/alfresco-search/generator-solr-config/generate-solr-config.py +++ b/search-services/alfresco-search/generator-solr-config/generate-solr-config.py @@ -66,10 +66,9 @@ def generate_fields(field_type, tokenized, string, cross_locale, sortable, sugge if string: generated_fields.append(get_copy_field_xml(field, create_non_tokenized(prefix))) + generated_fields.append(get_copy_field_xml(field, create_non_tokenized_cross_locale(prefix))) if sortable: generated_fields.append(get_copy_field_xml(field, create_sortable(prefix))) - if cross_locale: - generated_fields.append(get_copy_field_xml(field, create_non_tokenized_cross_locale(prefix))) if suggestable: generated_fields.append(get_copy_field_xml(field, "suggest")) @@ -107,7 +106,6 @@ def generate_text(file): file.write("\n") file.write("\n") - def main(): file = open(output_file, "w") file.write('\n') diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 3c84fda60..8631b46fc 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -280,6 +280,24 @@ src/main/resources/solr/instance + + conf/shared.properties + + + + + + + copy-test-conf + generate-test-resources + + copy-resources + + + ${project.build.testOutputDirectory}/test-files/conf + + + src/test/resources/test-files/conf diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java index c8281991a..360de7842 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java @@ -189,14 +189,7 @@ public class AlfrescoSolrHighlighter extends DefaultSolrHighlighter implements P { schemaFieldName = AlfrescoSolrDataModel.getInstance().mapProperty(localFieldName, FieldUse.FTS, request, 1); } - - // In case after the second query we didn't get the cross-locale version, then it's better to ignore the - // fieldMatch parameter; in this way we are sure the snippets will be properly returned (together with other - // unwanted fields) - if (!schemaFieldName.contains("_t@{")) - { - schemaFieldName = null; - } + // The query scorer purpose is to give a score to the text fragments by the number of unique query terms found. // diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties index 62dc5d66c..a80445145 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties @@ -16,19 +16,19 @@ alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}author alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner # Suggestable Propeties -alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name -alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title -alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description -alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content +#alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name +#alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title +#alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description +#alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content # Data types that support cross locale/word splitting/token patterns if tokenised alfresco.cross.locale.property.0={http://www.alfresco.org/model/content/1.0}name alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lockOwner # Data types that support cross locale/word splitting/token patterns if tokenised -alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text -alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content -alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext +#alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text +#alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content +#alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext alfresco.model.tracker.cron=0/10 * * * * ? * diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml index 7515129a5..7d743f35c 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml @@ -1,322 +1,346 @@ - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java index c72eb5e61..7b75204a0 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java @@ -295,7 +295,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { .collect(Collectors.toSet()); Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, - DynamicFieldType.TOKENIZED_LOCALE), dynamicFieldsTypes); + DynamicFieldType.TOKENIZED_LOCALE, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } @@ -340,7 +341,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { .collect(Collectors.toSet()); Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, - DynamicFieldType.SORT), dynamicFieldsTypes); + DynamicFieldType.SORT, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } @@ -361,7 +363,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, DynamicFieldType.TOKENIZED_LOCALE, - DynamicFieldType.SORT), dynamicFieldsTypes); + DynamicFieldType.SORT, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } @@ -424,7 +427,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, - DynamicFieldType.SORT), dynamicFieldsTypes); + DynamicFieldType.SORT, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } /** diff --git a/search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties b/search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties new file mode 100644 index 000000000..9c7acf8bb --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties @@ -0,0 +1,36 @@ + +# Shared Properties file + +#Host details an external client would use to connect to Solr +solr.host=localhost +#If not set then solr.port will be the jetty.port +#solr.port=8983 +solr.baseurl=/solr + +# Properties treated as identifiers when indexed + +alfresco.identifier.property.0={http://www.alfresco.org/model/content/1.0}creator +alfresco.identifier.property.1={http://www.alfresco.org/model/content/1.0}modifier +alfresco.identifier.property.2={http://www.alfresco.org/model/content/1.0}userName +alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}authorityName +alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner + +# Suggestable Propeties +alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name +alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title +alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description +alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content + +# Data types that support cross locale/word splitting/token patterns if tokenised +alfresco.cross.locale.property.0={http://www.alfresco.org/model/content/1.0}name +alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lockOwner + +# Data types that support cross locale/word splitting/token patterns if tokenised +alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text +alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content +alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext + +alfresco.model.tracker.cron=0/10 * * * * ? * + +# Whether path queries are enabled. +alfresco.cascade.tracker.enabled=true