From d1f124d931cea6cd2db4e5697a4cad64490c1946 Mon Sep 17 00:00:00 2001 From: elia Date: Thu, 3 Sep 2020 17:45:34 +0200 Subject: [PATCH 1/9] [SEARCH-2400] restored shared.properties as in 1.4.3 --- .../resources/solr/instance/conf/shared.properties | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties index 62dc5d66c..a80445145 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties @@ -16,19 +16,19 @@ alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}author alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner # Suggestable Propeties -alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name -alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title -alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description -alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content +#alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name +#alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title +#alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description +#alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content # Data types that support cross locale/word splitting/token patterns if tokenised alfresco.cross.locale.property.0={http://www.alfresco.org/model/content/1.0}name alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lockOwner # Data types that support cross locale/word splitting/token patterns if tokenised -alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text -alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content -alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext +#alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text +#alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content +#alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext alfresco.model.tracker.cron=0/10 * * * * ? * From 227df218f3fc5f4945bebd8e7eb34ed3aa3f6154 Mon Sep 17 00:00:00 2001 From: elia Date: Mon, 7 Sep 2020 12:43:04 +0200 Subject: [PATCH 2/9] [SEARCH-2400] changed music model --- e2e-test/src/main/resources/model/music-model.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/main/resources/model/music-model.xml b/e2e-test/src/main/resources/model/music-model.xml index d7de00d47..fc614da42 100644 --- a/e2e-test/src/main/resources/model/music-model.xml +++ b/e2e-test/src/main/resources/model/music-model.xml @@ -67,7 +67,7 @@ false BOTH - false + true From ae7fef1534acf63600bd83c3ddcb11486dc80612 Mon Sep 17 00:00:00 2001 From: elia Date: Mon, 7 Sep 2020 12:43:51 +0200 Subject: [PATCH 3/9] [SEARCH-2400] removed conf directory from solrhome in distributed tests --- .../src/test/java/org/alfresco/solr/SolrITInitializer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index 24d744f00..ef0c13e79 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -168,7 +168,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 RandomSupplier.RandVal.uniqueValues = new HashSet<>(); // reset random values createServers(testClassName, coreNames, numShards, solrcoreProperties); - + System.setProperty("solr.solr.home", testDir.toPath().resolve(testClassName).toString()); return testClassName; } @@ -531,8 +531,6 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { FileUtils.copyFile(new File(getTestFilesHome(), getSolrXml()), jettyHome.resolve(getSolrXml()).toFile()); - //Add solr home conf folder with alfresco based configuration. - FileUtils.copyDirectory(new File(getTestFilesHome() + "/conf"), jettyHome.resolve("conf").toFile()); // Add alfresco data model def FileUtils.copyDirectory(new File(getTestFilesHome() + "/alfrescoModels"), jettyHome.resolve("alfrescoModels").toFile()); // Add templates From 2daed8cab91b408e1c113747a55e13c17ec2cf0a Mon Sep 17 00:00:00 2001 From: elia Date: Tue, 8 Sep 2020 09:33:03 +0200 Subject: [PATCH 4/9] [SEARCH-2400] use different shared.properties for integration tests --- search-services/alfresco-search/pom.xml | 18 ++++++++++ .../org/alfresco/solr/SolrITInitializer.java | 2 ++ .../test-files/conf/shared.properties | 36 +++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 843e21555..fa2f0d4f9 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -280,6 +280,24 @@ src/main/resources/solr/instance + + conf/shared.properties + + + + + + + copy-test-conf + generate-test-resources + + copy-resources + + + ${project.build.testOutputDirectory}/test-files/conf + + + src/test/resources/test-files/conf diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index ef0c13e79..815e0b796 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -531,6 +531,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 { FileUtils.copyFile(new File(getTestFilesHome(), getSolrXml()), jettyHome.resolve(getSolrXml()).toFile()); + //Add solr home conf folder with alfresco based configuration. + FileUtils.copyDirectory(new File(getTestFilesHome() + "/conf"), jettyHome.resolve("conf").toFile()); // Add alfresco data model def FileUtils.copyDirectory(new File(getTestFilesHome() + "/alfrescoModels"), jettyHome.resolve("alfrescoModels").toFile()); // Add templates diff --git a/search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties b/search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties new file mode 100644 index 000000000..9c7acf8bb --- /dev/null +++ b/search-services/alfresco-search/src/test/resources/test-files/conf/shared.properties @@ -0,0 +1,36 @@ + +# Shared Properties file + +#Host details an external client would use to connect to Solr +solr.host=localhost +#If not set then solr.port will be the jetty.port +#solr.port=8983 +solr.baseurl=/solr + +# Properties treated as identifiers when indexed + +alfresco.identifier.property.0={http://www.alfresco.org/model/content/1.0}creator +alfresco.identifier.property.1={http://www.alfresco.org/model/content/1.0}modifier +alfresco.identifier.property.2={http://www.alfresco.org/model/content/1.0}userName +alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}authorityName +alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner + +# Suggestable Propeties +alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name +alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title +alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description +alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content + +# Data types that support cross locale/word splitting/token patterns if tokenised +alfresco.cross.locale.property.0={http://www.alfresco.org/model/content/1.0}name +alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lockOwner + +# Data types that support cross locale/word splitting/token patterns if tokenised +alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text +alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content +alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext + +alfresco.model.tracker.cron=0/10 * * * * ? * + +# Whether path queries are enabled. +alfresco.cascade.tracker.enabled=true From 4fd9c327798bce2601b769ba5248818d3e35cc34 Mon Sep 17 00:00:00 2001 From: elia Date: Wed, 9 Sep 2020 09:28:06 +0200 Subject: [PATCH 5/9] [SEARCH-2400] restored music model --- e2e-test/src/main/resources/model/music-model.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/main/resources/model/music-model.xml b/e2e-test/src/main/resources/model/music-model.xml index fc614da42..d7de00d47 100644 --- a/e2e-test/src/main/resources/model/music-model.xml +++ b/e2e-test/src/main/resources/model/music-model.xml @@ -67,7 +67,7 @@ false BOTH - true + false From 2f05094e5308f92d44c5794a8ce52a9081b8da3e Mon Sep 17 00:00:00 2001 From: elia Date: Wed, 9 Sep 2020 09:28:30 +0200 Subject: [PATCH 6/9] [SEARCh-2400] restored test to be the same as in 1.4.3 branch --- .../searchServices/search/SearchAFTSInFieldTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java index 6fc22c2f3..92130b0d3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java @@ -137,13 +137,13 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest boolean fileFound = isContentInSearchResults(query, file2.getName(), true); Assert.assertTrue(fileFound, "File Not found for query: " + query); - testSearchQuery(query, 2, SearchLanguage.AFTS); + testSearchQuery(query, 1, SearchLanguage.AFTS); query = fieldName + ":'" + file2.getName() + "\'"; fileFound = isContentInSearchResults(query, file2.getName(), true); Assert.assertTrue(fileFound, "File Not found for query: " + query); - testSearchQuery(query, 2, SearchLanguage.AFTS); + testSearchQuery(query, 1, SearchLanguage.AFTS); }); } From c6049dc1b8e867278495104e5cc6aa11bb341af6 Mon Sep 17 00:00:00 2001 From: elia Date: Wed, 9 Sep 2020 09:29:45 +0200 Subject: [PATCH 7/9] [SEARCH-2400] activate cross-locale field for all the non-tokenized properties --- .../generate-solr-config.py | 4 +- .../rerank/conf/generated_copy_fields.xml | 652 +++++++++--------- .../org/alfresco/solr/SolrITInitializer.java | 2 +- 3 files changed, 340 insertions(+), 318 deletions(-) diff --git a/search-services/alfresco-search/generator-solr-config/generate-solr-config.py b/search-services/alfresco-search/generator-solr-config/generate-solr-config.py index 927385845..4c4ffffb4 100644 --- a/search-services/alfresco-search/generator-solr-config/generate-solr-config.py +++ b/search-services/alfresco-search/generator-solr-config/generate-solr-config.py @@ -66,10 +66,9 @@ def generate_fields(field_type, tokenized, string, cross_locale, sortable, sugge if string: generated_fields.append(get_copy_field_xml(field, create_non_tokenized(prefix))) + generated_fields.append(get_copy_field_xml(field, create_non_tokenized_cross_locale(prefix))) if sortable: generated_fields.append(get_copy_field_xml(field, create_sortable(prefix))) - if cross_locale: - generated_fields.append(get_copy_field_xml(field, create_non_tokenized_cross_locale(prefix))) if suggestable: generated_fields.append(get_copy_field_xml(field, "suggest")) @@ -107,7 +106,6 @@ def generate_text(file): file.write("\n") file.write("\n") - def main(): file = open(output_file, "w") file.write('\n') diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml index 7515129a5..7d743f35c 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml @@ -1,322 +1,346 @@ - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index 815e0b796..24d744f00 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -168,7 +168,7 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 RandomSupplier.RandVal.uniqueValues = new HashSet<>(); // reset random values createServers(testClassName, coreNames, numShards, solrcoreProperties); - System.setProperty("solr.solr.home", testDir.toPath().resolve(testClassName).toString()); + return testClassName; } From 1cd13dbeb45343d41b6081d747f25c4fbae00ffe Mon Sep 17 00:00:00 2001 From: elia Date: Wed, 9 Sep 2020 11:41:26 +0200 Subject: [PATCH 8/9] [SEARCH-2400] fixed test on dynamic fields --- .../java/org/alfresco/solr/DynamicCopyFieldsIT.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java index c72eb5e61..7b75204a0 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/DynamicCopyFieldsIT.java @@ -295,7 +295,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { .collect(Collectors.toSet()); Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, - DynamicFieldType.TOKENIZED_LOCALE), dynamicFieldsTypes); + DynamicFieldType.TOKENIZED_LOCALE, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } @@ -340,7 +341,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { .collect(Collectors.toSet()); Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, - DynamicFieldType.SORT), dynamicFieldsTypes); + DynamicFieldType.SORT, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } @@ -361,7 +363,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, DynamicFieldType.TOKENIZED_LOCALE, - DynamicFieldType.SORT), dynamicFieldsTypes); + DynamicFieldType.SORT, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } @@ -424,7 +427,8 @@ public class DynamicCopyFieldsIT extends AbstractAlfrescoDistributedIT { Assert.assertEquals(Set.of(DynamicFieldType.STORED, DynamicFieldType.NON_TOKENIZED_LOCALE, - DynamicFieldType.SORT), dynamicFieldsTypes); + DynamicFieldType.SORT, + DynamicFieldType.NON_TOKENIZED_CROSS_LOCALE), dynamicFieldsTypes); } /** From 3ff76f197b21430e41cbc4855b1b0cc283a3740b Mon Sep 17 00:00:00 2001 From: elia Date: Thu, 10 Sep 2020 10:03:31 +0200 Subject: [PATCH 9/9] [SEARCH-2400] removed check for property with no cross-locale fields in highlighter --- .../solr/handler/component/AlfrescoSolrHighlighter.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java index c8281991a..360de7842 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java @@ -189,14 +189,7 @@ public class AlfrescoSolrHighlighter extends DefaultSolrHighlighter implements P { schemaFieldName = AlfrescoSolrDataModel.getInstance().mapProperty(localFieldName, FieldUse.FTS, request, 1); } - - // In case after the second query we didn't get the cross-locale version, then it's better to ignore the - // fieldMatch parameter; in this way we are sure the snippets will be properly returned (together with other - // unwanted fields) - if (!schemaFieldName.contains("_t@{")) - { - schemaFieldName = null; - } + // The query scorer purpose is to give a score to the text fragments by the number of unique query terms found. //