From f68a9912048cb0a43b87478ad65829b1db592224 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Mon, 29 Jul 2019 18:32:57 +0200 Subject: [PATCH 01/33] [ SEARCH-1717 ] review comment addressed --- .../doc-routers/00001-doc-routers.md | 26 +++++++++++++++++++ ...plicitShardIdWithStaticPropertyRouter.java | 5 ---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md b/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md index 7602ccafb..7c1117cd6 100644 --- a/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md +++ b/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md @@ -149,6 +149,19 @@ where ##### Last Registered Indexing Shard (LRIS) + +> WARNING: This is an experimental feature that is subject to change. +The current known issues are: +> +> \- Incompatibility with the "Purge" action on the Alfresco Admin Console +> \- Incompatibility with the "Purge on Startup" option on the Alfresco Admin Console +> +> As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription +> registration order. That would cause a wrong distribution of the indexed data across the cluster. +> +> Note the "Purge on restart" option on the Admin Console is checked by default, so before building a +> Solr cluster which uses this router, please make sure the check has been disabled. + This method uses, as the name suggests, the last indexing shard which subscribed to the Shard Registry. Like the _DB_ID_RANGE_ strategy, it's possible to add new shards to the cluster without requiring a reindex. And as consequence of that, the total number of shards (i.e. the _shard.count_ attribute) isn't needed with this sharding strategy. At indexing time, when the _MetadataTracker_ periodically asks for transactions and nodes, Alfresco repository creates the nodes instances and, using the associated transaction timestamp, asks to the _ShardRegistry_ which is the target Shard which should index/own the node. @@ -212,6 +225,19 @@ where * shard.count is the total number of the shards composing the cluster (required by the DBID fallback strategy) ##### Explicit Shard ID with fallback on LRIS (EXPLICIT_ID_FALLBACK_LRIS) + +> WARNING: This is an experimental feature that is subject to change. +The current known issues are: +> +> \- Incompatibility with the "Purge" action on the Alfresco Admin Console +> \- Incompatibility with the "Purge on Startup" option on the Alfresco Admin Console +> +> As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription +> registration order. That would cause a wrong distribution of the indexed data across the cluster. +> +> Note the "Purge on restart" option on the Admin Console is checked by default, so before building a +> Solr cluster which uses this router, please make sure the check has been disabled. + This method still belongs to the "explicit" id family, where nodes are routed to shards accordingly to a value of a node property which is supposed to contain the target shard instance identifier. So it is very similar to the previous one. The important difference resides in the fallback strategy: in case the target shard cannot be determined used the supplied node property, the "Last Registered Indexing Shard" is used as fallback. diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java index 9de9477ec..45ec3477f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java @@ -45,11 +45,6 @@ import org.slf4j.LoggerFactory; *

*

* - * Specifically, until the whole feature will be officially released, the LRIS document routing feature is not compatible - * with the "Purge" action on the Alfresco Admin Console. - * Note that at time of writing, the "Purge on startup" option in the Admin Console is enabled by default so prior to - * build your search cluster, you have to make sure that option is unchecked. - * * @author Elia * @author agazzarini * @since 1.4 From 501b3cc4507ded65013791f62642f5d38b8403f9 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Tue, 30 Jul 2019 09:40:40 +0100 Subject: [PATCH 02/33] Change TestGroup to be a class with string contants. Also tidy imports in classes where TestGroup is no longer used. --- .../java/org/alfresco/search/TestGroup.java | 58 +++++++++++-------- .../cmis/SolrSearchByAspectTests.java | 1 - .../cmis/SolrSearchByIdTests.java | 1 - .../cmis/SolrSearchByPathTests.java | 1 - .../cmis/SolrSearchByPropertyTests.java | 5 +- .../cmis/SolrSearchInFolderTests.java | 1 - .../cmis/SolrSearchInTreeTests.java | 1 - .../searchServices/sanity/SetupTest.java | 1 - .../search/FacetFieldsSearchTest.java | 6 +- .../search/FingerPrintTest.java | 7 +-- .../search/SearchHighLightTest.java | 1 - .../search/SearchWithCustomModelTest.java | 1 - .../solr/SearchSolrAPITest.java | 4 +- 13 files changed, 42 insertions(+), 46 deletions(-) diff --git a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java index 5b3cc475e..ae0cd7553 100644 --- a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java +++ b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java @@ -1,33 +1,43 @@ +/* + * Copyright (C) 2018 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ package org.alfresco.search; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target({ ElementType.METHOD }) -public @interface TestGroup +public class TestGroup { - // Used for TestRail test annotation - String SEARCH = "search"; - String REST_API = "rest-api"; + // Used for TestRail test annotation + public static final String SEARCH = "search"; + public static final String REST_API = "rest-api"; - String PREUPGRADE = "pre-upgrade"; - String POSTUPGRADE = "post-upgrade"; + public static final String PREUPGRADE = "pre-upgrade"; + public static final String POSTUPGRADE = "post-upgrade"; - String ASS_MASTER_SLAVE = "ASS_Master_Slave"; // Alfresco Search Services using master slave configurations - String ASS_MASTER ="ASS_Master"; // Alfresco search services using master/stand alone mode - String EXPLICIT_SHARDING ="Explicit_Sharding"; // Alfresco search services using sharded environment and explicit routing + public static final String ASS_MASTER_SLAVE = "ASS_Master_Slave"; // Alfresco Search Services using master slave configurations + public static final String ASS_MASTER ="ASS_Master"; // Alfresco search services using master/stand alone mode + public static final String EXPLICIT_SHARDING ="Explicit_Sharding"; // Alfresco search services using sharded environment and explicit routing - String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running + public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running - String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n - String ACS_60n = "ACS_60n"; // Alfresco Content Services 6.0 or above - String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above - String ACS_611n = "ACS_611n"; // Alfresco Content Services 6.1.1 or above - String ACS_62n = "ACS_62n"; // Alfresco Content Services 6.2 or above - - String NOT_BAMBOO = "Not_Bamboo"; // The does not run on bamboo + public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n + public static final String ACS_60n = "ACS_60n"; // Alfresco Content Services 6.0 or above + public static final String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above + public static final String ACS_611n = "ACS_611n"; // Alfresco Content Services 6.1.1 or above + public static final String ACS_62n = "ACS_62n"; // Alfresco Content Services 6.2 or above + public static final String NOT_BAMBOO = "Not_Bamboo"; // The does not run on bamboo } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java index 30b02e731..8e0e0eb08 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java @@ -7,7 +7,6 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; -import org.alfresco.search.TestGroup; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java index d10c31fa2..5d1026e29 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java @@ -7,7 +7,6 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; -import org.alfresco.search.TestGroup; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java index 0663d59fb..dc284c037 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java @@ -5,7 +5,6 @@ import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.provider.XMLTestData; import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.QueryModel; -import org.alfresco.search.TestGroup; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java index 70894ac57..40ea49483 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java @@ -1,13 +1,12 @@ package org.alfresco.test.search.functional.searchServices.cmis; import org.alfresco.utility.Utility; -import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.CustomObjectTypeProperties; +import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.provider.XMLTestDataProvider; -import org.alfresco.utility.model.QueryModel; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FolderModel; -import org.alfresco.search.TestGroup; +import org.alfresco.utility.model.QueryModel; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java index 5aebcb0f7..17647a6ed 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java @@ -7,7 +7,6 @@ import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; -import org.alfresco.search.TestGroup; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java index 125b3335e..3fd68cfa4 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java @@ -7,7 +7,6 @@ import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; import org.alfresco.utility.model.QueryModel; -import org.alfresco.search.TestGroup; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/sanity/SetupTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/sanity/SetupTest.java index b35d2db6d..01b8cdadb 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/sanity/SetupTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/sanity/SetupTest.java @@ -17,7 +17,6 @@ import org.alfresco.utility.data.DataSite; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; -import org.alfresco.search.TestGroup; import org.apache.chemistry.opencmis.commons.PropertyIds; import org.apache.chemistry.opencmis.commons.enums.VersioningState; import org.springframework.beans.factory.annotation.Autowired; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java index aa053aa30..7c0f4d129 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java @@ -18,11 +18,10 @@ */ package org.alfresco.test.search.functional.searchServices.search; -import java.util.ArrayList; -import java.util.List; - import javax.json.Json; import javax.json.JsonObject; +import java.util.ArrayList; +import java.util.List; import org.alfresco.dataprep.SiteService.Visibility; import org.alfresco.rest.search.FacetFieldBucket; @@ -32,7 +31,6 @@ import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.RestResultBucketsModel; import org.alfresco.rest.search.SearchRequest; import org.alfresco.rest.search.SearchResponse; -import org.alfresco.search.TestGroup; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.data.RandomData; import org.alfresco.utility.model.FileModel; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java index 77ebc165f..2f6f8b678 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java @@ -18,20 +18,19 @@ */ package org.alfresco.test.search.functional.searchServices.search; +import static org.testng.Assert.assertTrue; + import org.alfresco.rest.search.SearchNodeModel; import org.alfresco.rest.search.SearchResponse; +import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; -import org.alfresco.search.TestGroup; -import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.report.Bug; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import static org.testng.Assert.assertTrue; - /** * Search end point Public API test with finger print. * diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchHighLightTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchHighLightTest.java index 87073f439..da17ec049 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchHighLightTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchHighLightTest.java @@ -22,7 +22,6 @@ import org.alfresco.rest.search.RestRequestFieldsModel; import org.alfresco.rest.search.RestRequestHighlightModel; import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.SearchResponse; -import org.alfresco.search.TestGroup; import org.alfresco.utility.report.Bug; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchWithCustomModelTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchWithCustomModelTest.java index 7875f1680..f50c4174f 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchWithCustomModelTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchWithCustomModelTest.java @@ -18,7 +18,6 @@ import org.alfresco.utility.data.DataSite; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; -import org.alfresco.search.TestGroup; import org.apache.chemistry.opencmis.commons.PropertyIds; import org.apache.chemistry.opencmis.commons.enums.VersioningState; import org.springframework.beans.factory.annotation.Autowired; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/SearchSolrAPITest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/SearchSolrAPITest.java index a52bab254..1d7bc2522 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/SearchSolrAPITest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/SearchSolrAPITest.java @@ -14,14 +14,12 @@ */ package org.alfresco.test.search.functional.searchServices.solr; -import java.net.URLEncoder; - import javax.json.JsonArrayBuilder; +import java.net.URLEncoder; import org.alfresco.rest.core.JsonBodyGenerator; import org.alfresco.rest.model.RestTextResponse; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; -import org.alfresco.search.TestGroup; import org.hamcrest.Matchers; import org.springframework.http.HttpStatus; import org.testng.Assert; From 592e0cf48d3ff42c6d2809b2e5c17026f36a341a Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 1 Aug 2019 12:43:19 +0100 Subject: [PATCH 03/33] Refactor abstract test classes using changes to TAS. --- e2e-test/pom.xml | 6 +- .../functional/AbstractE2EFunctionalTest.java | 170 ++++++------------ .../cmis/AbstractCmisE2ETest.java | 3 +- .../cmis/SolrSearchScoreQueryTests.java | 2 +- .../search/AbstractSearchServicesE2ETest.java | 2 +- .../upgrade/AbstractUpgradeE2ETest.java | 2 +- 6 files changed, 66 insertions(+), 119 deletions(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 619a7f611..483f670a7 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,9 +11,9 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 6.0.0.3 - 6.0.0.1 - 2.0.12 + 6.0.1.2 + 6.0.0.4 + 2.0.31 2.6.0 src/test/resources/SearchSuite.xml diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index d85398287..06ba1f92d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -47,9 +47,10 @@ import static lombok.AccessLevel.PROTECTED; @ContextConfiguration("classpath:alfresco-search-e2e-context.xml") public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests { - public static final int SEARCH_MAX_ATTEMPS = 6; + /** The number of retries that a query will be tried before giving up. */ + private static final int SEARCH_MAX_ATTEMPTS = 6; - private static Logger LOG = LogFactory.getLogger(); + private static final Logger LOGGER = LogFactory.getLogger(); @Autowired protected RestProperties restProperties; @@ -85,8 +86,6 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont protected static String unique_searchString; - public static final String NODE_PREFIX = "workspace/SpacesStore/"; - @BeforeSuite(alwaysRun = true) public void beforeSuite() throws Exception { @@ -97,7 +96,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont } @BeforeClass(alwaysRun = true) - public void setup() throws Exception + public void setup() { serverHealth.assertServerIsOnline(); @@ -120,68 +119,44 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont if ((path != null) && (path.endsWith("-model.xml"))) { - try - { - dataContent.usingAdmin().deployContentModel(path); - modelDeployed = true; - } - catch (Exception e) - { - LOG.warn("Error Loading Custom Model", e); - } + dataContent.usingAdmin().deployContentModel(path); + modelDeployed = true; } return modelDeployed; } public boolean deactivateCustomModel(String fileName) { - Boolean modelDeactivated = false; + FileModel customModel = getCustomModel(fileName); - try + // Deactivate the model if found + if (customModel != null) { - FileModel customModel = getCustomModel(fileName); + cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).updateProperty("cm:modelActive", false); - // Deactivate the model if found - if (customModel != null) - { - - cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).updateProperty("cm:modelActive", false); - - modelDeactivated = true; - } + return true; } - catch (Exception e) - { - LOG.warn("Error Deactivating Custom Model", e); - } - return modelDeactivated; + return false; } public boolean deleteCustomModel(String fileName) { Boolean modelDeleted = false; - try + FileModel customModel = getCustomModel(fileName); + + // Delete the model if found + if (customModel != null) { - FileModel customModel = getCustomModel(fileName); + // cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).deleteContent(); + dataContent.usingAdmin().usingResource(customModel).deleteContent(); + restClient.authenticateUser(dataContent.getAdminUser()).withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(customModel); - // Delete the model if found - if (customModel != null) - { - // cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).deleteContent(); - dataContent.usingAdmin().usingResource(customModel).deleteContent(); - restClient.authenticateUser(dataContent.getAdminUser()).withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(customModel); - - modelDeleted = true; - } - else - { - LOG.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); - } + modelDeleted = true; } - catch (Exception e) + else { - LOG.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); + LOGGER.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); } return modelDeleted; @@ -191,32 +166,25 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont { FileModel customModel = null; - try - { - if ((fileName != null) && (fileName.endsWith("-model.xml"))) + if ((fileName != null) && (fileName.endsWith("-model.xml"))) + { + Session session = contentService.getCMISSession(dataUser.getAdminUser().getUsername(), dataUser.getAdminUser().getPassword()); + + CmisObject modelInRepo = session.getObjectByPath(String.format("/Data Dictionary/Models/%s", fileName)); + + if (modelInRepo != null) { - Session session = contentService.getCMISSession(dataUser.getAdminUser().getUsername(), dataUser.getAdminUser().getPassword()); - - CmisObject modelInRepo = session.getObjectByPath(String.format("/Data Dictionary/Models/%s", fileName)); - - if (modelInRepo != null) - { - customModel = new FileModel(modelInRepo.getName()); - customModel.setNodeRef(modelInRepo.getId()); - customModel.setNodeRef(customModel.getNodeRefWithoutVersion()); - customModel.setCmisLocation(String.format("/Data Dictionary/Models/%s", fileName)); - LOG.info("Custom Model file: " + customModel.getCmisLocation()); - } - else - { - LOG.info("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); - } + customModel = new FileModel(modelInRepo.getName()); + customModel.setNodeRef(modelInRepo.getId()); + customModel.setNodeRef(customModel.getNodeRefWithoutVersion()); + customModel.setCmisLocation(String.format("/Data Dictionary/Models/%s", fileName)); + LOGGER.info("Custom Model file: " + customModel.getCmisLocation()); + } + else + { + LOGGER.info("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); } - } - catch (Exception e) - { - LOG.warn("Error Getting Custom Model: " + fileName, e); } return customModel; @@ -224,29 +192,22 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Helper method which create an http post request to Search API end point. - * Executes the given search request without throwing checked exceptions (a {@link RuntimeException} will be thrown in case). + * Executes the given search request. * * @param query the search request. * @return the query execution response. */ protected SearchResponse query(SearchRequest query) { - try - { - return restClient.authenticateUser(testUser).withSearchAPI().search(query); - } - catch (final Exception exception) - { - throw new RuntimeException(exception); - } + return restClient.authenticateUser(testUser).withSearchAPI().search(query); } /** * Wait for Solr to finish indexing and search to return appropriate results * - * @param userQuery: Search Query + * @param userQuery Search Query * @param contentToFind that's expected to be included / excluded from the results - * @param expectedInResults + * @param expectedInResults Whether we expect the content in the results or not. * @return true if search returns expected results, i.e. is given content is found or excluded from the results */ public boolean isContentInSearchResults(String userQuery, String contentToFind, boolean expectedInResults) { @@ -254,30 +215,27 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont String expectedStatusCode = HttpStatus.OK.toString(); String contentName = (contentToFind == null) ? "" : contentToFind; + SearchRequest searchRequest = createQuery(userQuery); + // Repeat search until the query results are as expected or Search Retry count is hit - for (int searchCount = 0; searchCount < SEARCH_MAX_ATTEMPS; searchCount++) + for (int searchCount = 0; searchCount < SEARCH_MAX_ATTEMPTS; searchCount++) { - SearchRequest searchRequest = createQuery(userQuery); SearchResponse response = query(searchRequest); if (restClient.getStatusCode().matches(expectedStatusCode)) { - boolean found = response.getEntries().stream() .map(entry -> entry.getModel().getName()) .filter(name -> name.equalsIgnoreCase(contentName) || contentName.isBlank()) .count() > 0; - // Loop again if result is not as expected: To cater for solr lag: eventual consistency + // Exit loop if result is as expected. if (expectedInResults == found) { return true; } - else - { - // Wait for the solr indexing. - Utility.waitToLoopTime(properties.getSolrWaitTimeInSeconds(), "Wait For Indexing. Retry Attempt: " + searchCount); - } + // Wait for the solr indexing (eventual consistency). + Utility.waitToLoopTime(properties.getSolrWaitTimeInSeconds(), "Wait For Indexing. Retry Attempt: " + (searchCount + 1)); } else { @@ -294,9 +252,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont * @param userQuery: search query, this can include the fieldname, unique search string will guarantee accurate results * @param expectedInResults, true if entry is expected in the results set * @return true (indexing is finished) if search returns appropriate results - * @throws Exception */ - public boolean waitForIndexing(String userQuery, boolean expectedInResults) throws Exception + public boolean waitForIndexing(String userQuery, boolean expectedInResults) { // Use the search query as is: fieldname(s) may or may not be specified within the userQuery return waitForIndexing(null, userQuery, expectedInResults); @@ -308,9 +265,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont * @param userQuery * @param expectedInResults * @return - * @throws Exception */ - public boolean waitForMetadataIndexing(String userQuery, boolean expectedInResults) throws Exception + public boolean waitForMetadataIndexing(String userQuery, boolean expectedInResults) { return waitForIndexing("name", userQuery, expectedInResults); } @@ -322,9 +278,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont * @param userQuery * @param expectedInResults * @return - * @throws Exception */ - public boolean waitForContentIndexing(String userQuery, boolean expectedInResults) throws Exception + public boolean waitForContentIndexing(String userQuery, boolean expectedInResults) { return waitForIndexing("cm:content", userQuery, expectedInResults); } @@ -336,9 +291,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont * @param userQuery: search string, unique search string will guarantee accurate results * @param expectedInResults, true if entry is expected in the results set * @return true (indexing is finished) if search returns appropriate results - * @throws Exception */ - private boolean waitForIndexing(String fieldName, String userQuery, boolean expectedInResults) throws Exception + private boolean waitForIndexing(String fieldName, String userQuery, boolean expectedInResults) { String query = (fieldName == null) ? userQuery : String.format("%s:'%s'", fieldName, userQuery); @@ -365,16 +319,11 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont */ protected SearchResponse queryAsUser(UserModel user, String queryString) { - try { - SearchRequest searchRequest = new SearchRequest(); - RestRequestQueryModel queryModel = new RestRequestQueryModel(); - queryModel.setQuery(queryString); - searchRequest.setQuery(queryModel); - return restClient.authenticateUser(user).withSearchAPI().search(searchRequest); - } catch (final Exception exception) - { - throw new RuntimeException(exception); - } + SearchRequest searchRequest = new SearchRequest(); + RestRequestQueryModel queryModel = new RestRequestQueryModel(); + queryModel.setQuery(queryString); + searchRequest.setQuery(queryModel); + return restClient.authenticateUser(user).withSearchAPI().search(searchRequest); } /** @@ -384,7 +333,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont * @param queryModel: The queryModel to search for, containing the query * @return the search response from the API */ - protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel) throws Exception + protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel) { SearchRequest searchRequest = new SearchRequest(); searchRequest.setQuery(queryModel); @@ -396,9 +345,8 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont * Helper method which create an http post request to Search API end point. * * @return {@link SearchResponse} response. - * @throws Exception if error */ - protected SearchResponse query(RestRequestQueryModel queryReq, RestRequestHighlightModel highlight) throws Exception + protected SearchResponse query(RestRequestQueryModel queryReq, RestRequestHighlightModel highlight) { SearchRequest query = new SearchRequest(queryReq); query.setHighlight(highlight); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java index 6f31ec0ac..3dc7ed8c3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java @@ -57,9 +57,8 @@ public abstract class AbstractCmisE2ETest extends AbstractE2EFunctionalTest * @param query CMIS Query to be executed * @param expectedCountResults Number of results expected * @return true when results count is equals to expectedCountResults - * @throws Exception */ - protected boolean waitForIndexing(String query, long expectedCountResults) throws Exception + protected boolean waitForIndexing(String query, long expectedCountResults) { for (int searchCount = 1; searchCount <= 3; searchCount++) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java index 36ad1eabe..85b597fea 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java @@ -43,7 +43,7 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest * @see org.alfresco.test.search.functional.AbstractE2EFunctionalTest#dataPreparation() */ @Override - public void setup() throws Exception + public void setup() { } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/AbstractSearchServicesE2ETest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/AbstractSearchServicesE2ETest.java index edd3c230a..2beaa33f4 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/AbstractSearchServicesE2ETest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/AbstractSearchServicesE2ETest.java @@ -35,7 +35,7 @@ public abstract class AbstractSearchServicesE2ETest extends AbstractE2EFunctiona protected FileModel file, file2, file3, file4; - public void searchServicesDataPreparation() throws Exception + public void searchServicesDataPreparation() { /* * Create the following file structure for preconditions : diff --git a/e2e-test/src/test/java/org/alfresco/test/search/nonFunctional/upgrade/AbstractUpgradeE2ETest.java b/e2e-test/src/test/java/org/alfresco/test/search/nonFunctional/upgrade/AbstractUpgradeE2ETest.java index f3959727e..59f618e9f 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/nonFunctional/upgrade/AbstractUpgradeE2ETest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/nonFunctional/upgrade/AbstractUpgradeE2ETest.java @@ -37,7 +37,7 @@ public abstract class AbstractUpgradeE2ETest extends AbstractTestNGSpringContext protected XMLTestData testData; @BeforeClass(alwaysRun = true) - public void checkServerHealth() throws Exception + public void checkServerHealth() { serverHealth.assertServerIsOnline(); } From 5e58da0afb42de8114d679c103ae332d6de66d32 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Thu, 1 Aug 2019 15:03:34 +0100 Subject: [PATCH 04/33] SEARCH-1742 Bump TAS utility version. --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 483f670a7..b264bff23 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 6.0.1.2 6.0.0.4 - 2.0.31 + 2.0.32 2.6.0 src/test/resources/SearchSuite.xml From 2245ba00e70fcda3af477b75c400761edbfcd3e6 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Thu, 1 Aug 2019 16:32:32 +0200 Subject: [PATCH 05/33] [ SEARCH-1666 ] Formal refactor according with Alfresco formatting rules --- ...butedDateMonthAlfrescoSolrTrackerTest.java | 111 +++++++++--------- 1 file changed, 53 insertions(+), 58 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java index ed5e49fb2..8e87d103b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java @@ -18,6 +18,17 @@ */ package org.alfresco.solr.tracker; +import static java.util.Collections.singletonList; +import static java.util.stream.IntStream.range; +import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; + import org.alfresco.model.ContentModel; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; @@ -33,7 +44,6 @@ import org.alfresco.solr.client.StringPropertyValue; import org.alfresco.solr.client.Transaction; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; import org.junit.AfterClass; @@ -49,40 +59,23 @@ import java.util.List; import java.util.Properties; import java.util.TimeZone; -import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE; -import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; -import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; -import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; -import static org.alfresco.solr.AlfrescoSolrUtils.getNode; -import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; -import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; -import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; -import static org.alfresco.solr.AlfrescoSolrUtils.list; - -/** - * @author Joel - * - * - * - * - */ - @SolrTestCaseJ4.SuppressSSL -@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"}) public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfrescoDistributedTest { @BeforeClass - private static void initData() throws Throwable + @SuppressWarnings("unused") + public static void initData() throws Throwable { - initSolrServers(5, "DistributedDateMonthAlfrescoSolrTrackerTest", getShardMethod()); + initSolrServers(5, DistributedDateMonthAlfrescoSolrTrackerTest.class.getSimpleName(), getShardMethod()); } @AfterClass - private static void destroyData() throws Throwable + @SuppressWarnings("unused") + public static void destroyData() { dismissSolrServers(); } - + @Test public void testDateMonth() throws Exception { @@ -91,58 +84,57 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc int numAcls = 25; AclChangeSet bulkAclChangeSet = getAclChangeSet(numAcls); - List bulkAcls = new ArrayList(); - List bulkAclReaders = new ArrayList(); + List bulkAcls = new ArrayList<>(); + List bulkAclReaders = new ArrayList<>(); - - for (int i = 0; i < numAcls; i++) { + for (int i = 0; i < numAcls; i++) + { Acl bulkAcl = getAcl(bulkAclChangeSet); bulkAcls.add(bulkAcl); bulkAclReaders.add(getAclReaders(bulkAclChangeSet, bulkAcl, - list("joel" + bulkAcl.getId()), - list("phil" + bulkAcl.getId()), + singletonList("joel" + bulkAcl.getId()), + singletonList("phil" + bulkAcl.getId()), null)); } - indexAclChangeSet(bulkAclChangeSet, - bulkAcls, - bulkAclReaders); + indexAclChangeSet(bulkAclChangeSet, bulkAcls, bulkAclReaders); int numNodes = 1000; - List nodes = new ArrayList(); - List nodeMetaDatas = new ArrayList(); + List nodes = new ArrayList<>(); + List nodeMetaData = new ArrayList<>(); Transaction bigTxn = getTransaction(0, numNodes); - Date[] dates = new Date[5]; + Calendar calendar = new GregorianCalendar(); + calendar.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar cal = new GregorianCalendar(); - cal.setTimeZone(TimeZone.getTimeZone("UTC")); - for (int i = 0; i < dates.length; i++) { - - cal.set(1980, i, 21); - dates[i] = cal.getTime(); - } + Date[] dates = range(0, 5) + .peek(index -> calendar.set(1980, index, 21)) + .mapToObj(index -> calendar.getTime()) + .toArray(Date[]::new); int[] counts = new int[dates.length]; - for (int i = 0; i < numNodes; i++) { + for (int i = 0; i < numNodes; i++) + { int aclIndex = i % numAcls; int dateIndex = i % dates.length; String dateString = DefaultTypeConverter.INSTANCE.convert(String.class, dates[dateIndex]); + counts[dateIndex]++; + Node node = getNode(bigTxn, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED); node.setShardPropertyValue(dateString); nodes.add(node); - NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "mike", null, false); - nodeMetaData.getProperties().put(ContentModel.PROP_CREATED, - new StringPropertyValue(dateString)); - nodeMetaDatas.add(nodeMetaData); + NodeMetaData metadata = getNodeMetaData(node, bigTxn, bulkAcls.get(aclIndex), "mike", null, false); + metadata.getProperties().put(ContentModel.PROP_CREATED, new StringPropertyValue(dateString)); + + nodeMetaData.add(metadata); } - indexTransaction(bigTxn, nodes, nodeMetaDatas); + indexTransaction(bigTxn, nodes, nodeMetaData); waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes, 100000); waitForDocCountAllCores(new TermQuery(new Term(FIELD_DOC_TYPE, SolrInformationServer.DOC_TYPE_ACL)), numAcls, 100000); @@ -150,7 +142,6 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0); String fieldName = fieldInstance.getField(); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); format.setTimeZone(TimeZone.getTimeZone("GMT")); for (int i = 0; i < dates.length; i++) @@ -161,32 +152,36 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc gcal.add(Calendar.SECOND, 1); String endDate = format.format(gcal.getTime()); - SolrQuery solrQuery = new SolrQuery("{!lucene}" + escapeQueryChars(fieldName) + - ":[" + escapeQueryChars(startDate) + " TO " + escapeQueryChars(endDate) + " } " ); + SolrQuery solrQuery = + new SolrQuery("{!lucene}" + escapeQueryChars(fieldName) + ":[" + escapeQueryChars(startDate) + " TO " + escapeQueryChars(endDate) + " } " ); + assertCountAndColocation(solrQuery, counts[i]); assertShardSequence(i, solrQuery, counts[i]); } nodes.clear(); - nodeMetaDatas.clear(); + nodeMetaData.clear(); Transaction bigTxn1 = getTransaction(0, numNodes); - for (int i = 0; i < numNodes; i++) { + for (int i = 0; i < numNodes; i++) + { int aclIndex = i % numAcls; Node node = getNode(bigTxn1, bulkAcls.get(aclIndex), Node.SolrApiNodeStatus.UPDATED); nodes.add(node); - NodeMetaData nodeMetaData = getNodeMetaData(node, bigTxn1, bulkAcls.get(aclIndex), "mike", null, false); - nodeMetaDatas.add(nodeMetaData); + + NodeMetaData metadata = getNodeMetaData(node, bigTxn1, bulkAcls.get(aclIndex), "mike", null, false); + nodeMetaData.add(metadata); } - indexTransaction(bigTxn1, nodes, nodeMetaDatas); + indexTransaction(bigTxn1, nodes, nodeMetaData); waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), numNodes*2, 100000); + //There are 5 shards. We should expect roughly 20% of the nodes on each shard assertNodesPerShardGreaterThan((int)((numNodes*2)*.17)); } - protected static Properties getShardMethod() + private static Properties getShardMethod() { Properties prop = new Properties(); prop.put("shard.method", ShardMethodEnum.DATE.toString()); From 590763b38bb8ee8d3e895388057714376e745071 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2019 00:17:49 +0000 Subject: [PATCH 06/33] Bump alfresco-data-model from 8.45 to 8.46 in /search-services Bumps [alfresco-data-model](https://github.com/Alfresco/alfresco-data-model) from 8.45 to 8.46. - [Release notes](https://github.com/Alfresco/alfresco-data-model/releases) - [Commits](https://github.com/Alfresco/alfresco-data-model/compare/8.45...8.46) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 6484927c6..8dfd03a31 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -22,7 +22,7 @@ - 8.45 + 8.46 2.10.0.pr1 From 17201507a37600143570e0796238bd3b04781ffd Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 2 Aug 2019 08:22:51 +0100 Subject: [PATCH 07/33] SEARCH-1742 Update TAS Utility to 3.0.x. Also fix some issues in previous commit around exception handling. --- e2e-test/pom.xml | 3 +- .../functional/AbstractE2EFunctionalTest.java | 92 ++++++++++++------- .../cmis/SolrSearchByPathTests.java | 4 +- 3 files changed, 64 insertions(+), 35 deletions(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index b264bff23..f388149d1 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 6.0.1.2 6.0.0.4 - 2.0.32 + 3.0.9 2.6.0 src/test/resources/SearchSuite.xml @@ -45,6 +45,7 @@ ${test.exclude} ${test.include} + true diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index 06ba1f92d..2f259adb0 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -119,22 +119,36 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont if ((path != null) && (path.endsWith("-model.xml"))) { - dataContent.usingAdmin().deployContentModel(path); - modelDeployed = true; + try + { + dataContent.usingAdmin().deployContentModel(path); + modelDeployed = true; + } + catch (Exception e) + { + LOGGER.warn("Error Loading Custom Model", e); + } } return modelDeployed; } public boolean deactivateCustomModel(String fileName) { - FileModel customModel = getCustomModel(fileName); - - // Deactivate the model if found - if (customModel != null) + try { - cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).updateProperty("cm:modelActive", false); + FileModel customModel = getCustomModel(fileName); - return true; + // Deactivate the model if found + if (customModel != null) + { + cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).updateProperty("cm:modelActive", false); + + return true; + } + } + catch (Exception e) + { + LOGGER.warn("Error Deactivating Custom Model", e); } return false; } @@ -143,18 +157,25 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont { Boolean modelDeleted = false; - FileModel customModel = getCustomModel(fileName); - - // Delete the model if found - if (customModel != null) + try { - // cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).deleteContent(); - dataContent.usingAdmin().usingResource(customModel).deleteContent(); - restClient.authenticateUser(dataContent.getAdminUser()).withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(customModel); + FileModel customModel = getCustomModel(fileName); - modelDeleted = true; + // Delete the model if found + if (customModel != null) + { + // cmisApi.authenticateUser(dataUser.getAdminUser()).usingResource(customModel).deleteContent(); + dataContent.usingAdmin().usingResource(customModel).deleteContent(); + restClient.authenticateUser(dataContent.getAdminUser()).withCoreAPI().usingTrashcan().deleteNodeFromTrashcan(customModel); + + modelDeleted = true; + } + else + { + LOGGER.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); + } } - else + catch (Exception e) { LOGGER.error("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); } @@ -166,25 +187,32 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont { FileModel customModel = null; - if ((fileName != null) && (fileName.endsWith("-model.xml"))) - + try { - Session session = contentService.getCMISSession(dataUser.getAdminUser().getUsername(), dataUser.getAdminUser().getPassword()); + if ((fileName != null) && (fileName.endsWith("-model.xml"))) - CmisObject modelInRepo = session.getObjectByPath(String.format("/Data Dictionary/Models/%s", fileName)); - - if (modelInRepo != null) { - customModel = new FileModel(modelInRepo.getName()); - customModel.setNodeRef(modelInRepo.getId()); - customModel.setNodeRef(customModel.getNodeRefWithoutVersion()); - customModel.setCmisLocation(String.format("/Data Dictionary/Models/%s", fileName)); + Session session = contentService.getCMISSession(dataUser.getAdminUser().getUsername(), dataUser.getAdminUser().getPassword()); + + CmisObject modelInRepo = session.getObjectByPath(String.format("/Data Dictionary/Models/%s", fileName)); + + if (modelInRepo != null) + { + customModel = new FileModel(modelInRepo.getName()); + customModel.setNodeRef(modelInRepo.getId()); + customModel.setNodeRef(customModel.getNodeRefWithoutVersion()); + customModel.setCmisLocation(String.format("/Data Dictionary/Models/%s", fileName)); LOGGER.info("Custom Model file: " + customModel.getCmisLocation()); + } + else + { + LOGGER.info("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); + } } - else - { - LOGGER.info("Custom Content Model [{}] is not available under [/Data Dictionary/Models/] location", fileName); - } + } + catch (Exception e) + { + LOGGER.warn("Error Getting Custom Model: " + fileName, e); } return customModel; @@ -192,7 +220,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont /** * Helper method which create an http post request to Search API end point. - * Executes the given search request. + * Executes the given search request without throwing checked exceptions (a {@link RuntimeException} will be thrown in case). * * @param query the search request. * @return the query execution response. diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java index dc284c037..c1e0ae1b9 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java @@ -38,7 +38,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getAllData") @XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml") - public void prepareDataForSearchByPath(XMLTestData testData) throws Exception + public void prepareDataForSearchByPath(XMLTestData testData) { this.testData = testData; testData.createUsers(dataUser); @@ -49,7 +49,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") @XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml") - public void executeSearchByPathQueries(QueryModel query) throws Exception + public void executeSearchByPathQueries(QueryModel query) { cmisApi.withQuery(query.getValue()).assertResultsCount().equals(query.getResults()); } From 26b5d2a4bb953520856d8728b20a907d7a30e8be Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 2 Aug 2019 13:55:02 +0200 Subject: [PATCH 08/33] [ SEARCH-1717 ] review comment addressed (II) --- .../solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java index 45ec3477f..87bf6deb5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithStaticPropertyRouter.java @@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory; * As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription * registration order. That would cause a wrong distribution of the indexed data across the cluster. * - * Note the "Purge on restart" option on the Admin Console is checked by default, so before building a + * Note that if "Purge on restart" option on the Admin Console is checked by default, then before building a * Solr cluster which uses this router, please make sure the check has been disabled. * *

From 045325ba183ff5be89ba6a62a37da659410cf5ca Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 2 Aug 2019 13:56:45 +0200 Subject: [PATCH 09/33] [ SEARCH-1717 ] review comment addressed on ADR --- .../doc/architecture/doc-routers/00001-doc-routers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md b/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md index 7c1117cd6..b9c097b3d 100644 --- a/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md +++ b/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md @@ -235,7 +235,7 @@ The current known issues are: > As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription > registration order. That would cause a wrong distribution of the indexed data across the cluster. > -> Note the "Purge on restart" option on the Admin Console is checked by default, so before building a +> Note that if "Purge on restart" option on the Admin Console is checked by default, then before building a > Solr cluster which uses this router, please make sure the check has been disabled. This method still belongs to the "explicit" id family, where nodes are routed to shards accordingly to a value of a node property which is supposed to contain the target shard instance identifier. From 408c2cd1973d16b5169f7ddfacaa2ced88a76c02 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Fri, 2 Aug 2019 13:58:09 +0200 Subject: [PATCH 10/33] [ SEARCH-1717 ] review comment addressed on ADR --- .../doc/architecture/doc-routers/00001-doc-routers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md b/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md index b9c097b3d..998377263 100644 --- a/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md +++ b/search-services/alfresco-search/doc/architecture/doc-routers/00001-doc-routers.md @@ -159,7 +159,7 @@ The current known issues are: > As a result, when this router is used and the purge action is invoked you may experience a wrong shard subscription > registration order. That would cause a wrong distribution of the indexed data across the cluster. > -> Note the "Purge on restart" option on the Admin Console is checked by default, so before building a +> Note that if "Purge on restart" option on the Admin Console is checked by default, then before building a > Solr cluster which uses this router, please make sure the check has been disabled. This method uses, as the name suggests, the last indexing shard which subscribed to the Shard Registry. From 6d69399d7c443c6106bbe6667cdb26c6f2d85ef4 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 2 Aug 2019 13:41:19 +0100 Subject: [PATCH 11/33] SEARCH-1742 Check for float != null rather than empty. It doesn't really make sense to check if a float is empty, although the TAS library seems to allow checking if integers and longs are empty for some reason. --- .../search/functional/searchServices/search/SearchTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java index b3771d4a8..aa49cf8e5 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java @@ -72,7 +72,7 @@ public class SearchTest extends AbstractSearchServicesE2ETest SearchNodeModel entity = nodes.getEntryByIndex(0); entity.assertThat().field("search").contains("score"); - entity.getSearch().assertThat().field("score").isNotEmpty(); + entity.getSearch().assertThat().field("score").isNotNull(); Assert.assertEquals(entity.getName(),"pangram.txt"); } From abbef3ce1d5d7eccae1728f7e4f50a6c81ec4fad Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 2 Aug 2019 14:08:46 +0100 Subject: [PATCH 12/33] SEARCH-1742 Use txt extension so that document content is indexed correctly. --- .../functional/searchServices/search/FingerPrintTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java index 2f6f8b678..858e8b2f5 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java @@ -79,7 +79,7 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest public void makeSureFingerprintQueryWorksAfterMetadataUpdate() throws Exception { // Index a new file with content - FileModel file = new FileModel("Project_Contract.pdf", FileType.TEXT_PLAIN, "A content which is completely different from other indexed files."); + FileModel file = new FileModel("Project_Contract.txt", FileType.TEXT_PLAIN, "A content which is completely different from other indexed files."); dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file); // make sure the content has been indexed (i.e. the ContentTracker fingerprint has been correctly computed From 8d290e35f4ac87900a7df0f200f27d2f55ef8af7 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Mon, 5 Aug 2019 15:56:40 +0200 Subject: [PATCH 13/33] [ SEARCH-1666 ] Fix on date formatter --- .../tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java index 8e87d103b..4e4c03541 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedDateMonthAlfrescoSolrTrackerTest.java @@ -42,6 +42,7 @@ import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.StringPropertyValue; import org.alfresco.solr.client.Transaction; +import org.alfresco.util.CachingDateFormat; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; import org.apache.solr.SolrTestCaseJ4; @@ -142,8 +143,7 @@ public class DistributedDateMonthAlfrescoSolrTrackerTest extends AbstractAlfresc AlfrescoSolrDataModel.FieldInstance fieldInstance = fieldInstanceList.get(0); String fieldName = fieldInstance.getField(); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - format.setTimeZone(TimeZone.getTimeZone("GMT")); + SimpleDateFormat format = CachingDateFormat.getSolrDatetimeFormatWithoutMsecs(); for (int i = 0; i < dates.length; i++) { String startDate = format.format(dates[i]); From 50a757eb44fe3497048870545d1a8420580ead68 Mon Sep 17 00:00:00 2001 From: michaelsuzukisagi Date: Tue, 6 Aug 2019 09:34:47 +0100 Subject: [PATCH 14/33] SEARCH-1677, added component diagram --- .../componenet-diagram.puml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 search-services/alfresco-search/doc/architecture/AsyncContentTracker/componenet-diagram.puml diff --git a/search-services/alfresco-search/doc/architecture/AsyncContentTracker/componenet-diagram.puml b/search-services/alfresco-search/doc/architecture/AsyncContentTracker/componenet-diagram.puml new file mode 100644 index 000000000..03aa5c54e --- /dev/null +++ b/search-services/alfresco-search/doc/architecture/AsyncContentTracker/componenet-diagram.puml @@ -0,0 +1,33 @@ +@startuml + +database "Solr" { + package "SolrInfrmationServer" { + [UpdateIndex] + } + package "org.alfresco.tracker.content"{ + [ContentConsumer] + [ContentMessageProducer] + [SharedFileStoreClient] + } + folder "ContentStore" { + [UpdateFile] + } +} +node "Alfresco SharedFileStore" { + [GetFile] +} + +database "ActiveMQ" { +[ContentMessageQue] +} + +[ContentMessageProducer] --> [ContentMessageQue] +[ContentConsumer] --> [ContentMessageQue] +[ContentConsumer] --> [SharedFileStoreClient] +[ContentConsumer] <-- [SharedFileStoreClient] +[SharedFileStoreClient] --> [GetFile] +[ContentConsumer] --> [UpdateIndex] +[UpdateIndex] -->[UpdateFile] + + +@enduml \ No newline at end of file From 7d65a61e4ee59fee87267892f050ad29bf16454b Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 6 Aug 2019 12:34:59 +0200 Subject: [PATCH 15/33] Support for AGS latest (based in ACS 6.1) --- .../README.md | 17 +++++++- .../generators/app/index.js | 40 ++++++++++++++++--- .../generators/app/templates/6.1/.env | 9 ++++- .../app/templates/6.1/docker-compose-ce.yml | 2 +- .../app/templates/6.1/docker-compose-ee.yml | 2 +- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/e2e-test/generator-alfresco-docker-compose/README.md b/e2e-test/generator-alfresco-docker-compose/README.md index e7a0f445f..664e12ab9 100644 --- a/e2e-test/generator-alfresco-docker-compose/README.md +++ b/e2e-test/generator-alfresco-docker-compose/README.md @@ -16,9 +16,11 @@ Following templates are provided. $ tree generators/app/templates/ generators/app/templates/ ├── 6.1 +│   ├── .env │   ├── docker-compose-ce.yml │   └── docker-compose-ee.yml ├── 6.2 +│   ├── .env │   ├── docker-compose-ce.yml │   └── docker-compose-ee.yml ├── images @@ -62,14 +64,24 @@ $ yo alfresco-docker-compose ## ACS Version -Currently supported ACS Version are `6.2` and `6.1` +Currently supported ACS Versions are `6.2` and `6.1` This is the first choice to be selected when the generator is executed. + ``` ? Which ACS version do you want to use? 6.1 ❯ 6.2 ``` +## AGS Version + +Currently supported AGS Version is `latest` (based in ACS 6.1). + +If you chose ACS 6.1, a prompt will allow you to use AGS. + +``` +? Would you like to use AGS? Yes +``` ## Community @@ -122,6 +134,8 @@ Shard property is `shard:shardId`, belonging to a custom model deployed in the t **EXPLICIT_ID** and **EXPLICIT_ID_FALLBACK_LRIS** Custom content model is deployed to provide a property, named `shard:shardId`, holding the Shard Number (0, 1) where the content is indexed. +This default configuration can be changed in the generated `docker-compose.yml` template. + ## Configuration catalog | Version | Comms | Replication | Sharding | Explicit | Insight | Zeppelin | @@ -152,6 +166,7 @@ $ yo alfresco-docker-compose --acsVersion=6.2 --alfrescoVersion=community --http **Parameter names reference** `--acsVersion`: default 6.2, but could be set to 6.1 +`--ags:`: only available when acsVersion=6.1 `--alfrescoVersion`: community or enterprise `--httpMode`: http or https `--clustering`: true or false diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js index d8c9d6007..5cd8f1fd9 100644 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js @@ -28,6 +28,13 @@ module.exports = class extends Generator { choices: [ "6.1", "6.2" ], default: '6.2' }, + { + whenFunction: response => response.acsVersion == '6.1', + type: 'confirm', + name: 'ags', + message: 'Would you like to use AGS?', + default: false + }, { type: 'list', name: 'alfrescoVersion', @@ -122,9 +129,13 @@ module.exports = class extends Generator { } // Docker Compose environment variables values - this.fs.copy( + this.fs.copyTpl( this.templatePath(dockerComposeTemplateDirectory + '/.env'), - this.destinationPath('.env') + this.destinationPath('.env'), + { + acs: (this.props.ags ? "false" : "true"), + ags: (this.props.ags ? "true" : "false") + } ) // Base Docker Compose Template @@ -136,8 +147,26 @@ module.exports = class extends Generator { // Repository Docker Image name const acsImageName = (this.props.alfrescoVersion == 'community' ? - 'alfresco/alfresco-content-repository-community' : - 'alfresco/alfresco-content-repository'); + (this.props.ags ? + 'alfresco/alfresco-governance-repository-community' : + 'alfresco/alfresco-content-repository-community') : + (this.props.ags ? + 'quay.io/alfresco/alfresco-governance-repository-enterprise': + 'alfresco/alfresco-content-repository' + ) + ); + + // Share Docker Image name + const shareImageName = + (this.props.alfrescoVersion == 'community' ? + (this.props.ags ? + 'alfresco/alfresco-governance-share-community' : + 'alfresco/alfresco-share') : + (this.props.ags ? + 'quay.io/alfresco/alfresco-governance-share-enterprise': + 'alfresco/alfresco-share' + ) + ); // Search Docker Image const searchImageName = @@ -164,7 +193,8 @@ module.exports = class extends Generator { searchPath: searchBasePath, zeppelin: (this.props.zeppelin ? "true" : "false"), sharding: (this.props.sharding ? "true" : "false"), - shardingMethod: (this.props.shardingMethod) + shardingMethod: (this.props.shardingMethod), + shareImage: shareImageName } ); diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env index c95352d50..969be3121 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env @@ -1,6 +1,11 @@ +<% if (acs == 'true') { %> ALFRESCO_TAG=6.1.0.5 -ALFRESCO_CE_TAG=6.1.2-ga -SHARE_TAG=6.1.0 +ALFRESCO_CE_TAG=6.1.2-ga +SHARE_TAG=6.1.0 <% } %> +<% if (ags == 'true') { %> +ALFRESCO_TAG=latest +ALFRESCO_CE_TAG=latest +SHARE_TAG=latest <% } %> POSTGRES_TAG=10.1 TRANSFORM_ROUTER_TAG=1.0.1 PDF_RENDERER_TAG=2.0.10 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml index 663ce09bf..e7342c754 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml @@ -124,7 +124,7 @@ services: <% } %> share: - image: alfresco/alfresco-share:${SHARE_TAG} + image: <%=shareImage%>:${SHARE_TAG} mem_limit: 1g environment: REPO_HOST: "alfresco" diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml index 8c9d074f7..3ae6e144d 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml @@ -264,7 +264,7 @@ services: - shared-file-store-volume:/tmp/Alfresco/sfs share: - image: alfresco/alfresco-share:${SHARE_TAG} + image: <%=shareImage%>:${SHARE_TAG} mem_limit: 1g environment: REPO_HOST: "alfresco" From d130d0525271ff2f611a537819a45b5ee3d9046c Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 6 Aug 2019 16:57:26 +0200 Subject: [PATCH 16/33] Enable custom model deployments for repository and custom forms deployments to Share. --- .../README.md | 38 ++++++++++++++++++- .../generators/app/index.js | 34 +++++++++++++++-- .../app/templates/6.1/docker-compose-ce.yml | 5 ++- .../app/templates/6.1/docker-compose-ee.yml | 5 ++- .../app/templates/6.2/docker-compose-ce.yml | 5 ++- .../app/templates/6.2/docker-compose-ee.yml | 5 ++- .../app/templates/images/alfresco/Dockerfile | 6 +-- .../app/templates/images/alfresco/model/empty | 0 .../alfresco/model/sharding-content-model.xml | 2 + .../app/templates/images/share/Dockerfile | 7 ++++ .../templates/images/share/model/empty.xml | 5 +++ .../model/sharding-share-config-custom.xml | 27 +++++++++++++ 12 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/empty create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/Dockerfile create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/empty.xml create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/sharding-share-config-custom.xml diff --git a/e2e-test/generator-alfresco-docker-compose/README.md b/e2e-test/generator-alfresco-docker-compose/README.md index 664e12ab9..db4254c68 100644 --- a/e2e-test/generator-alfresco-docker-compose/README.md +++ b/e2e-test/generator-alfresco-docker-compose/README.md @@ -27,20 +27,32 @@ generators/app/templates/ │   ├── alfresco │   │   ├── Dockerfile │   │   └── model +│   │   ├── empty │   │   ├── sharding-content-model-context.xml │   │   └── sharding-content-model.xml │   ├── search │   │   └── Dockerfile +│   ├── share +│   │   ├── Dockerfile +│   │   └── model +│   │   ├── empty.xml +│   │   └── sharding-share-config-custom.xml │   └── zeppelin │   └── Dockerfile └── keystores + ├── alfresco + ├── client + ├── solr + └── zeppelin ``` * `.env` includes default values for Docker Compose environment variables * `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for ACS 6.2 and ACS 6.1) * `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for ACS 6.2 and ACS 6.1) * `alfresco` includes a Dockerfile template to start Alfresco Repository - * `model` includes a default content model for Sharding Explicit Routing + * `model` includes a default content model (Sharding Explicit Routing or empty) +* `share` includes a Dockerfile template to start Share Web Application + * `model` includes a default forms model (Sharding Explicit Routing or empty) * `search` includes a Dockerfile template to start Search Services and Insight Engine * `zeppelin` includes a Dockerfile template to start Zeppelin with SSL * `keystores` includes every truststore and keystore required for SSL configuration @@ -136,6 +148,30 @@ Custom content model is deployed to provide a property, named `shard:shardId`, h This default configuration can be changed in the generated `docker-compose.yml` template. + +## Deploying custom content models and forms + +Custom content models and Share Form configurations can be added to deployment folders. + +**Content models** + +Custom content models can be copied to Repository deployment folder by using bootstrap approach. + +Following XML files must be created in `alfresco/model/` folder in the Docker Compose template generated: + +* `content-model.xml` including an XML Alfresco Content Model file. Sample model is available in [images/alfresco/model/sharding-content-model.xml](generators/app/templates/images/alfresco/model/sharding-content-model.xml) +* `content-model-context.xml` including an XML Spring Bean file with the `dictionaryBootstrap` bean. Sample Spring Bean declaration is available in [images/alfresco/model/sharding-content-model-context.xml](generators/app/templates/images/alfresco/model/sharding-content-model-context.xml) + +If *Sharding* is selected, these files will be available in deployment folder. + +**Share forms** + +Custom content forms can be added to Share configuration by modifying `share/model/share-config-custom-dev.xml` file in the Docker Compose template generated. + +Sample configuration is available in [images/share/model/sharding-share-config-custom.xml](generators/app/templates/images/share/model/sharding-share-config-custom.xml) + +If *Sharding* is selected, a default `share-config-custom-dev.xml` file with required forms configuration for Sharding custom model will be available in deployment folder. Add your configuration to this file. + ## Configuration catalog | Version | Comms | Replication | Sharding | Explicit | Insight | Zeppelin | diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js index 5cd8f1fd9..72e6df1b8 100644 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js @@ -193,8 +193,7 @@ module.exports = class extends Generator { searchPath: searchBasePath, zeppelin: (this.props.zeppelin ? "true" : "false"), sharding: (this.props.sharding ? "true" : "false"), - shardingMethod: (this.props.shardingMethod), - shareImage: shareImageName + shardingMethod: (this.props.shardingMethod) } ); @@ -209,10 +208,37 @@ module.exports = class extends Generator { sharding: (this.props.sharding ? "true" : "false") } ); + // Copy Sharding Content Model or an empty file to allow model deployments if (this.props.sharding) { this.fs.copy( - this.templatePath(imagesDirectory + '/alfresco/model'), - this.destinationPath('alfresco/model') + this.templatePath(imagesDirectory + '/alfresco/model/*.xml'), + this.destinationPath('alfresco/model/') + ) + } else { + this.fs.copy( + this.templatePath(imagesDirectory + '/alfresco/model/empty'), + this.destinationPath('alfresco/model/empty') + ) + } + + // Copy Docker Image for Share applying configuration + this.fs.copyTpl( + this.templatePath(imagesDirectory + '/share/Dockerfile'), + this.destinationPath('share/Dockerfile'), + { + shareImage: shareImageName + } + ); + // Copy Sharding Content Forms or an empty file to allow forms deployments + if (this.props.sharding) { + this.fs.copy( + this.templatePath(imagesDirectory + '/share/model/sharding-share-config-custom.xml'), + this.destinationPath('share/model/share-config-custom-dev.xml') + ) + } else { + this.fs.copy( + this.templatePath(imagesDirectory + '/share/model/empty.xml'), + this.destinationPath('share/model/share-config-custom-dev.xml') ) } diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml index e7342c754..6b3d721b3 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml @@ -124,7 +124,10 @@ services: <% } %> share: - image: <%=shareImage%>:${SHARE_TAG} + build: + context: ./share + args: + SHARE_TAG: ${SHARE_TAG} mem_limit: 1g environment: REPO_HOST: "alfresco" diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml index 3ae6e144d..4eb2cee8a 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml @@ -264,7 +264,10 @@ services: - shared-file-store-volume:/tmp/Alfresco/sfs share: - image: <%=shareImage%>:${SHARE_TAG} + build: + context: ./share + args: + SHARE_TAG: ${SHARE_TAG} mem_limit: 1g environment: REPO_HOST: "alfresco" diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml index 903170400..602d8c756 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml @@ -178,7 +178,10 @@ services: - 8094:8090 share: - image: alfresco/alfresco-share:${SHARE_TAG} + build: + context: ./share + args: + SHARE_TAG: ${SHARE_TAG} mem_limit: 1g environment: REPO_HOST: "alfresco" diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml index 384cfa9a2..de15ad6bb 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml @@ -287,7 +287,10 @@ services: - shared-file-store-volume:/tmp/Alfresco/sfs share: - image: alfresco/alfresco-share:${SHARE_TAG} + build: + context: ./share + args: + SHARE_TAG: ${SHARE_TAG} mem_limit: 1g environment: REPO_HOST: "alfresco" diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile index da62587cc..a5595c7ac 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile @@ -48,7 +48,5 @@ RUN if [ "$SOLR_COMMS" == "https" ] ; then \ <\/Connector>/g" ${TOMCAT_DIR}/conf/server.xml; \ fi -<% if (sharding == 'true') { %> -# Explicit Routing Sharding content model (useless when not using this feature) -COPY model/sharding-content-model*.xml $TOMCAT_DIR/shared/classes/alfresco/extension/ -<% } %> \ No newline at end of file +# Copy custom content model to deployment folder +COPY model/* $TOMCAT_DIR/shared/classes/alfresco/extension/ diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/empty b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/empty new file mode 100644 index 000000000..e69de29bb diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/sharding-content-model.xml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/sharding-content-model.xml index 628f58eb6..159d02f12 100644 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/sharding-content-model.xml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/model/sharding-content-model.xml @@ -13,9 +13,11 @@ + Sharding + Shard Id d:text diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/Dockerfile b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/Dockerfile new file mode 100644 index 000000000..26fcbedd9 --- /dev/null +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/Dockerfile @@ -0,0 +1,7 @@ +ARG SHARE_TAG +FROM <%=shareImage%>:${SHARE_TAG} + +ARG TOMCAT_DIR=/usr/local/tomcat + +# Copy custom content forms to deployment folder +COPY model/* $TOMCAT_DIR/shared/classes/alfresco/web-extension/ \ No newline at end of file diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/empty.xml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/empty.xml new file mode 100644 index 000000000..8d1410bab --- /dev/null +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/empty.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/sharding-share-config-custom.xml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/sharding-share-config-custom.xml new file mode 100644 index 000000000..89a785108 --- /dev/null +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/model/sharding-share-config-custom.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + +
+ + + +
+ + + + \ No newline at end of file From e3e98a701f089ce6be27982db2cd85ebb04192b4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2019 00:14:32 +0000 Subject: [PATCH 17/33] Bump slf4j-log4j12 from 1.7.26 to 1.7.27 in /search-services Bumps [slf4j-log4j12](https://github.com/qos-ch/slf4j) from 1.7.26 to 1.7.27. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.26...v_1.7.27) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- search-services/alfresco-solrclient-lib/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 46e141f77..91257be71 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -81,7 +81,7 @@ org.slf4j slf4j-log4j12 - 1.7.26 + 1.7.27 diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 8dfd03a31..4e25f20b0 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -73,7 +73,7 @@ org.slf4j slf4j-log4j12 - 1.7.26 + 1.7.27 From a3fc5614251f06a84613787f81dcd19c8f094909 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2019 00:15:17 +0000 Subject: [PATCH 18/33] Bump slf4j-api from 1.7.26 to 1.7.27 in /search-services Bumps [slf4j-api](https://github.com/qos-ch/slf4j) from 1.7.26 to 1.7.27. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.26...v_1.7.27) Signed-off-by: dependabot-preview[bot] --- search-services/alfresco-search/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 46e141f77..b21262b8c 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -75,7 +75,7 @@ org.slf4j slf4j-api - 1.7.26 + 1.7.27 From 2ced2fa24c003a9083debb634889dfefccef7622 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2019 00:24:52 +0000 Subject: [PATCH 19/33] Bump utility from 3.0.9 to 3.0.11 in /e2e-test Bumps [utility](https://github.com/Alfresco/alfresco-tas-utility) from 3.0.9 to 3.0.11. - [Release notes](https://github.com/Alfresco/alfresco-tas-utility/releases) - [Changelog](https://github.com/Alfresco/alfresco-tas-utility/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/Alfresco/alfresco-tas-utility/compare/utility-3.0.9...utility-3.0.11) Signed-off-by: dependabot-preview[bot] --- e2e-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index f388149d1..b0d9d7147 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -13,7 +13,7 @@ 6.0.1.2 6.0.0.4 - 3.0.9 + 3.0.11 2.6.0 src/test/resources/SearchSuite.xml From 8d2488873bde4651ff686242b175ebe0776f9cab Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Wed, 7 Aug 2019 11:35:31 +0200 Subject: [PATCH 20/33] Describing what "empty" file does. --- e2e-test/generator-alfresco-docker-compose/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/generator-alfresco-docker-compose/README.md b/e2e-test/generator-alfresco-docker-compose/README.md index db4254c68..4014c7d1d 100644 --- a/e2e-test/generator-alfresco-docker-compose/README.md +++ b/e2e-test/generator-alfresco-docker-compose/README.md @@ -50,7 +50,7 @@ generators/app/templates/ * `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for ACS 6.2 and ACS 6.1) * `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for ACS 6.2 and ACS 6.1) * `alfresco` includes a Dockerfile template to start Alfresco Repository - * `model` includes a default content model (Sharding Explicit Routing or empty) + * `model` includes a default content model (Sharding Explicit Routing or empty). This `empty` file is required for Dockerfile to work, so it should not be deleted. * `share` includes a Dockerfile template to start Share Web Application * `model` includes a default forms model (Sharding Explicit Routing or empty) * `search` includes a Dockerfile template to start Search Services and Insight Engine From b2068a36ffbd0430f63adf18989f114853773626 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Thu, 8 Aug 2019 15:08:04 +0200 Subject: [PATCH 21/33] Adding "-a" parameter to documentation --- search-services/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/README.md b/search-services/README.md index 7b2322395..2cf51f776 100644 --- a/search-services/README.md +++ b/search-services/README.md @@ -100,7 +100,7 @@ SOLR_SSL_WANT_CLIENT_AUTH=false Once this settings are ready, start SOLR service from command line: ``` -$ ./solr/bin/solr start "-Dcreate.alfresco.defaults=alfresco,archive \ +$ ./solr/bin/solr start -a "-Dcreate.alfresco.defaults=alfresco,archive \ -Dsolr.ssl.checkPeerName=false \ -Dsolr.allow.unsafe.resourceloading=true" -f ``` @@ -130,7 +130,7 @@ DBID based sharding can be set up from the command line. For example a core cont index with twelve shards can be set up by starting an instance of Alfresco Search Services with a command like: ```bash -./bin/solr start -a -Dcreate.alfresco.defaults=alfresco,archive -Dnum.shards=12 -Dshard.ids=0,1,6,7 +./bin/solr start -a "-Dcreate.alfresco.defaults=alfresco,archive -Dnum.shards=12 -Dshard.ids=0,1,6,7" ``` Further instances should be set up to contain the other shards, and it is possible to adjust the distribution and @@ -169,7 +169,7 @@ $ sed -i 's/alfresco.secureComms=https/alfresco.secureComms=none/' solrhome/temp Start SOLR service from command line: ``` -$ ./solr/bin/solr start "-Dcreate.alfresco.defaults=alfresco,archive" -f +$ ./solr/bin/solr start -a "-Dcreate.alfresco.defaults=alfresco,archive" -f ``` SOLR will create Alfresco cores (`alfresco` and `archive`) when starting, and configuration from `rerank` template will be copied to each core and if you also started an ACS instance running in [http://localhost:8080/alfresco](http://localhost:8080/alfresco) then the index will be populated. From c294e9b3e70fcd56033021acf521a75a7c2cadc5 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Thu, 8 Aug 2019 16:14:05 +0200 Subject: [PATCH 22/33] Including proxy to 6.1 CE to server both alfresco and share web apps in port 8080 --- .../README.md | 29 +++++++++---------- .../generators/app/templates/6.1/.env | 1 + .../app/templates/6.1/docker-compose-ce.yml | 17 +++++++---- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/e2e-test/generator-alfresco-docker-compose/README.md b/e2e-test/generator-alfresco-docker-compose/README.md index 4014c7d1d..e78cfd477 100644 --- a/e2e-test/generator-alfresco-docker-compose/README.md +++ b/e2e-test/generator-alfresco-docker-compose/README.md @@ -174,20 +174,17 @@ If *Sharding* is selected, a default `share-config-custom-dev.xml` file with req ## Configuration catalog -| Version | Comms | Replication | Sharding | Explicit | Insight | Zeppelin | -| - | - | - | - | - | - | - | -| community | http | - | - | - | x | x | -| community | http | true | x | x | x | x | -| community | http | false | true | false | x | x | -| community | http | false | true | true | x | x | -| community | https | x | - | - | x | x | -| enterprise | http | - | - | - | (*) | (*) | -| enterprise | http | true | x | x | (*) | (*) | -| enterprise | http | false | true | false | (*) | (*) | -| enterprise | http | false | true | true | (*) | (*) | -| enterprise | https | x | - | - | (*) | (*) | -| enterprise | https | x | true | false | (*) | (*) | -| enterprise | https | x | true | true | (*) | (*) | +| Version | Comms | Replication | Sharding | Insight | Zeppelin | +| - | - | - | - | - | - | +| community | http | - | x | x | x | +| community | http | true | x | x | x | +| community | https | x | x | x | x | +| enterprise | http | - | - | (*) | (*) | +| enterprise | http | true | false | (*) | (*) | +| enterprise | http | false | true | (*) | (*) | +| enterprise | https | x | - | (*) | (*) | +| enterprise | https | x | true | (*) | (*) | +| enterprise | https | x | true | (*) | (*) | Both `community` and `enterprise` ACS deployments can be used with the same options, but `enterprise` may also use Insight Engine (replacing Search Services) and Insight Zeppelin services. @@ -231,7 +228,7 @@ $ docker-compose down http://localhost:8080/share -http://localhost:8082/alfresco +http://localhost:8080/alfresco http://localhost:8083/solr @@ -244,7 +241,7 @@ http://localhost:8084/solr http://localhost:8080/share -http://localhost:8082/alfresco +http://localhost:8080/alfresco https://localhost:8443/alfresco diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env index 969be3121..5b9a74a4c 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/.env @@ -16,5 +16,6 @@ SHARED_FILE_STORE_TAG=0.5.3 ACTIVE_MQ_TAG=5.15.8 DIGITAL_WORKSPACE_TAG=1.1.0 ACS_NGINX_TAG=3.0.0 +ACS_COMMUNITY_NGINX_TAG=1.0.0 SEARCH_TAG=latest ZEPPELIN_TAG=latest diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml index 6b3d721b3..24f66e6df 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml @@ -32,10 +32,8 @@ services: -Ddeployment.method=DOCKER_COMPOSE -Dcsrf.filter.enabled=false -Dalfresco.restApi.basicAuthScheme=true - -Xms1500m -Xmx1500m - " + " <% if (httpMode == 'https') { %> ports: - - 8082:8080 <% if (httpMode == 'https') { %> - 8443:8443 volumes: - ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %> @@ -140,8 +138,6 @@ services: -Dalfresco.context=alfresco -Dalfresco.protocol=http " - ports: - - 8080:8080 postgres: image: postgres:${POSTGRES_TAG} @@ -162,3 +158,14 @@ services: - 5672:5672 # AMQP - 61616:61616 # OpenWire - 61613:61613 # STOMP + + proxy: + image: alfresco/acs-community-ngnix:${ACS_COMMUNITY_NGINX_TAG} + mem_limit: 128m + depends_on: + - alfresco + ports: + - 8080:8080 + links: + - alfresco + - share \ No newline at end of file From d743cec51abb6f5473a816b19ef7bf57557eeca5 Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 9 Aug 2019 08:29:42 +0100 Subject: [PATCH 23/33] Update version to 1.5.0-SNAPSHOT. --- pom.xml | 2 +- search-services/alfresco-search/pom.xml | 4 ++-- search-services/alfresco-solrclient-lib/pom.xml | 2 +- search-services/packaging/pom.xml | 2 +- search-services/pom.xml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index d319a8824..264d4b963 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 10 alfresco-search-and-insight-parent - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT pom Alfresco Search And Insight Parent diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 99df50afe..67bca4acd 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -6,7 +6,7 @@ org.alfresco alfresco-search-parent - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml @@ -15,7 +15,7 @@ org.alfresco alfresco-solrclient-lib - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT servlet-api diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 4e25f20b0..cac0093a8 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-search-parent - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 30fe95987..b618f5365 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -13,7 +13,7 @@ org.alfresco alfresco-search-parent - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml diff --git a/search-services/pom.xml b/search-services/pom.xml index 02dd65c80..87be9a59c 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -4,13 +4,13 @@ org.alfresco alfresco-search-and-insight-parent - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT org.alfresco alfresco-search-parent - 1.4.0-SNAPSHOT + 1.5.0-SNAPSHOT pom Alfresco Solr Search parent From a564d2af987400cff2bee9879d4ed525500d1e3c Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 9 Aug 2019 12:44:28 +0100 Subject: [PATCH 24/33] SEARCH-1799 Check for shard property again if not found at start up. --- .../solr/tracker/MetadataTracker.java | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 794cdf7aa..059dfc2de 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -78,6 +78,9 @@ public class MetadataTracker extends AbstractTracker implements Tracker private ConcurrentLinkedQueue nodesToPurge = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue(); private DocRouter docRouter; + /** The string representation of the shard key. */ + private String shardKey; + /** The property to use for determining the shard. */ private QName shardProperty; public MetadataTracker(Properties p, SOLRAPIClient client, String coreName, @@ -86,15 +89,24 @@ public class MetadataTracker extends AbstractTracker implements Tracker super(p, client, coreName, informationServer, Tracker.Type.MetaData); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); - String shardKey = p.getProperty(DocRouterFactory.SHARD_KEY_KEY); - if(shardKey != null) { - shardProperty = getShardProperty(shardKey); - } + shardKey = p.getProperty(DocRouterFactory.SHARD_KEY_KEY); + updateShardProperty(); docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); } - + + /** + * Set the shard property using the shard key. + */ + private void updateShardProperty() + { + if(shardKey != null) + { + shardProperty = getShardProperty(shardKey); + } + } + MetadataTracker() { super(Tracker.Type.MetaData); @@ -224,6 +236,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker HashMap propertyBag = new HashMap<>(); propertyBag.put("coreName", coreName); HashMap extendedPropertyBag = new HashMap<>(propertyBag); + updateShardProperty(); extendedPropertyBag.putAll(docRouter.getProperties(shardProperty)); return ShardStateBuilder.shardState() @@ -365,6 +378,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); + updateShardProperty(); gnp.setShardProperty(shardProperty); gnp.setCoreName(coreName); @@ -885,6 +899,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker gnp.setTransactionIds(txIds); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); + updateShardProperty(); gnp.setShardProperty(shardProperty); gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, Integer.MAX_VALUE); @@ -1185,7 +1200,8 @@ public class MetadataTracker extends AbstractTracker implements Tracker this.queriesToReindex.offer(query); } - public static QName getShardProperty(String field) { + public static QName getShardProperty(String field) + { AlfrescoSolrDataModel dataModel = AlfrescoSolrDataModel.getInstance(); NamespaceDAO namespaceDAO = dataModel.getNamespaceDAO(); DictionaryService dictionaryService = dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT); @@ -1194,6 +1210,11 @@ public class MetadataTracker extends AbstractTracker implements Tracker dictionaryService, field); + if (propertyDef == null) + { + log.error("Sharding property not found: {}", field); + return null; + } return propertyDef.getName(); } } From b3a7096a5cbdc9f4a8672f791c09c5fcc5a8740f Mon Sep 17 00:00:00 2001 From: Tom Page Date: Fri, 9 Aug 2019 13:59:36 +0100 Subject: [PATCH 25/33] SEARCH-1799 Only look up shard property if not already found. --- .../main/java/org/alfresco/solr/tracker/MetadataTracker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 059dfc2de..7b5c08f2a 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -101,7 +101,7 @@ public class MetadataTracker extends AbstractTracker implements Tracker */ private void updateShardProperty() { - if(shardKey != null) + if(shardProperty == null && shardKey != null) { shardProperty = getShardProperty(shardKey); } From ed603af738a38f2398a7c13d723895ff48fbe9a9 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Mon, 12 Aug 2019 17:10:22 +0200 Subject: [PATCH 26/33] Support for ACS 5.2.6-RC2 --- search-services/packaging/src/docker/5.x/.env | 6 +- .../src/docker/5.x/docker-compose.yml | 90 ++++++++++++------- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/search-services/packaging/src/docker/5.x/.env b/search-services/packaging/src/docker/5.x/.env index 2c267dcd9..179e09887 100644 --- a/search-services/packaging/src/docker/5.x/.env +++ b/search-services/packaging/src/docker/5.x/.env @@ -1,2 +1,4 @@ -ALFRESCO_TAG=5.2.5-SNAPSHOT -SEARCH_TAG=latest \ No newline at end of file +ALFRESCO_TAG=5.2.6-RC2 +SHARE_TAG=5.2.6-RC2 +SEARCH_TAG=latest +POSTGRES_TAG=9.4 diff --git a/search-services/packaging/src/docker/5.x/docker-compose.yml b/search-services/packaging/src/docker/5.x/docker-compose.yml index e72d9c683..9a29ac981 100644 --- a/search-services/packaging/src/docker/5.x/docker-compose.yml +++ b/search-services/packaging/src/docker/5.x/docker-compose.yml @@ -1,33 +1,59 @@ -version: '3' +# Using version 2 as 3 does not support resource constraint options (cpu_*, mem_* limits) for non swarm mode in Compose +version: "2" + services: - platform: - image: docker-internal.alfresco.com/platform:${ALFRESCO_TAG} - environment: - JAVA_TOOL_OPTIONS: " - -Dsolr.host=search - -Dsolr.port=8983 - -Dsolr.secureComms=none - -Dsolr.base.url=/solr - -Dindex.subsystem.name=solr6 - " - ports: - - "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi - - "5005:5005" #Java debugging - - "8081:8080" #Browser port for Alfresco - - "8082:8080" #Browser port for Share - - "8084:8080" #Browser port for API Explorer - search: - image: quay.io/alfresco/search-services:${SEARCH_TAG} - environment: - #Solr needs to know how to register itself with Alfresco - SOLR_ALFRESCO_HOST: platform - SOLR_ALFRESCO_PORT: 8080 - #Alfresco needs to know how to call solr - SOLR_SOLR_HOST: search - SOLR_SOLR_PORT: 8983 - #Create the default alfresco and archive cores - SOLR_CREATE_ALFRESCO_DEFAULTS: alfresco,archive - #HTTP by default - ALFRESCO_SECURE_COMMS: none - ports: - - "8083:8983" #Browser port for Solr \ No newline at end of file + alfresco: + image: quay.io/alfresco/alfresco-content-repository-52:${ALFRESCO_TAG} + mem_limit: 2g + environment: + MEM_LIMIT: 2200m + JAVA_TOOL_OPTIONS: " + -Dsolr.host=search + -Dsolr.port=8983 + -Dsolr.secureComms=none + -Dsolr.base.url=/solr + -Dindex.subsystem.name=solr6 + " + ports: + - 7203:7203 #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi + - 5005:5005 #Java debugging + - 8081:8080 #Browser port for Alfresco + + share: + image: quay.io/alfresco/alfresco-share-52:${SHARE_TAG} + mem_limit: 1g + environment: + MEM_LIMIT: 1200m + ports: + - 8082:8080 + + search: + image: quay.io/alfresco/search-services:${SEARCH_TAG} + environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: alfresco + SOLR_ALFRESCO_PORT: 8080 + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: search + SOLR_SOLR_PORT: 8983 + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: alfresco,archive + #HTTP by default + ALFRESCO_SECURE_COMMS: none + ports: + - 8083:8983 + + libreoffice: + image: xcgd/libreoffice + mem_limit: 1g + + postgres: + image: postgres:${POSTGRES_TAG} + mem_limit: 512m + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + command: postgres -c max_connections=300 -c log_min_messages=LOG + ports: + - 5432:5432 From 3b990f52460691f924e48f1ffea12f6825ce0565 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 13 Aug 2019 15:40:22 +0200 Subject: [PATCH 27/33] Removed additional properties from documentation. --- search-services/README.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/search-services/README.md b/search-services/README.md index 2cf51f776..3f72c054d 100644 --- a/search-services/README.md +++ b/search-services/README.md @@ -247,14 +247,7 @@ $ docker run -p 8983:8983 \ -e SOLR_SSL_TRUST_STORE_TYPE=JCEKS \ -e SOLR_SSL_NEED_CLIENT_AUTH=true \ -e SOLR_OPTS="-Dsolr.ssl.checkPeerName=false \ --Dsolr.allow.unsafe.resourceloading=true \ --Dalfresco.encryption.ssl.keystore.type=JCEKS --Dalfresco.encryption.ssl.keystore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.keystore --Dalfresco.encryption.ssl.keystore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-keystore-passwords.properties --Dalfresco.encryption.ssl.truststore.type=JCEKS --Dalfresco.encryption.ssl.truststore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.truststore --Dalfresco.encryption.ssl.truststore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-truststore-passwords.properties -" \ +-Dsolr.allow.unsafe.resourceloading=true" \ searchservices:develop ``` @@ -345,12 +338,6 @@ solr6: SOLR_OPTS: " -Dsolr.ssl.checkPeerName=false -Dsolr.allow.unsafe.resourceloading=true - -Dalfresco.encryption.ssl.keystore.type=JCEKS - -Dalfresco.encryption.ssl.keystore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.keystore - -Dalfresco.encryption.ssl.keystore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-keystore-passwords.properties - -Dalfresco.encryption.ssl.truststore.type=JCEKS - -Dalfresco.encryption.ssl.truststore.location=/opt/alfresco-search-services/keystores/ssl.repo.client.truststore - -Dalfresco.encryption.ssl.truststore.passwordFileLocation=/opt/alfresco-search-services/keystores/ssl-truststore-passwords.properties " ports: - 8083:8983 #Browser port From a41f6c7846861adcee02e9903b66a9ea658f1276 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Thu, 15 Aug 2019 12:25:21 +0200 Subject: [PATCH 28/33] [ SEARCH-1804 ] use logger for test messages --- .../java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java index d545f43e6..7fcef7193 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedTest.java @@ -89,7 +89,7 @@ import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSI @ThreadLeakLingering(linger = 5000) public abstract class AbstractAlfrescoDistributedTest extends SolrTestInitializer { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected String[] deadServers; protected static SolrResponsesComparator solrComparator = new SolrResponsesComparator(); From 81a4717c5100d0b592e1df8765c580df5d0c87c4 Mon Sep 17 00:00:00 2001 From: bamboo_auth Date: Mon, 19 Aug 2019 10:41:13 +0000 Subject: [PATCH 29/33] [maven-release-plugin] prepare release 1.1.0 --- pom.xml | 4 ++-- search-services/alfresco-search/pom.xml | 4 ++-- search-services/alfresco-solrclient-lib/pom.xml | 2 +- search-services/packaging/pom.xml | 2 +- search-services/pom.xml | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 264d4b963..3b2e0f8c0 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 10 alfresco-search-and-insight-parent - 1.5.0-SNAPSHOT + 1.1.0 pom Alfresco Search And Insight Parent @@ -24,7 +24,7 @@ scm:git:https://git.alfresco.com/search_discovery/insightengine.git scm:git:https://git.alfresco.com/search_discovery/insightengine.git https://git.alfresco.com/search_discovery/insightengine.git - HEAD + 1.1.0 6.6.5 diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 67bca4acd..74ec70be5 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -6,7 +6,7 @@ org.alfresco alfresco-search-parent - 1.5.0-SNAPSHOT + 1.1.0 ../pom.xml @@ -15,7 +15,7 @@ org.alfresco alfresco-solrclient-lib - 1.5.0-SNAPSHOT + 1.1.0 servlet-api diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index cac0093a8..0a13db91a 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-search-parent - 1.5.0-SNAPSHOT + 1.1.0 diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index b618f5365..d7ce0529a 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -13,7 +13,7 @@ org.alfresco alfresco-search-parent - 1.5.0-SNAPSHOT + 1.1.0 ../pom.xml diff --git a/search-services/pom.xml b/search-services/pom.xml index 87be9a59c..62910a456 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -4,13 +4,13 @@ org.alfresco alfresco-search-and-insight-parent - 1.5.0-SNAPSHOT + 1.1.0 org.alfresco alfresco-search-parent - 1.5.0-SNAPSHOT + 1.1.0 pom Alfresco Solr Search parent From 2dadf882d90e5d5c03d023b89268c04a373ba09a Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 22 Aug 2019 12:47:26 +0200 Subject: [PATCH 30/33] [SEARCH-960] ModelTracker refactoring --- .../alfresco/solr/tracker/ModelTracker.java | 99 +++++++------------ 1 file changed, 37 insertions(+), 62 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 687fb4077..cc0d2577f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -83,18 +83,18 @@ import org.json.JSONException; public class ModelTracker extends AbstractTracker implements Tracker { - private Set indexedStores = new HashSet(); - private Set ignoredStores = new HashSet(); - private Set indexedTenants = new HashSet(); - private Set ignoredTenants = new HashSet(); - private Set indexedDataTypes = new HashSet(); - private Set ignoredDataTypes = new HashSet(); - private Set indexedTypes = new HashSet(); - private Set ignoredTypes = new HashSet(); - private Set indexedAspects = new HashSet(); - private Set ignoredAspects = new HashSet(); - private Set indexedFields = new HashSet(); - private Set ignoredFields = new HashSet(); + private final Set indexedStores = new HashSet<>(); + private final Set ignoredStores = new HashSet<>(); + private final Set indexedTenants = new HashSet<>(); + private final Set ignoredTenants = new HashSet<>(); + private final Set indexedDataTypes = new HashSet<>(); + private final Set ignoredDataTypes = new HashSet<>(); + private final Set indexedTypes = new HashSet<>(); + private final Set ignoredTypes = new HashSet<>(); + private final Set indexedAspects = new HashSet<>(); + private final Set ignoredAspects = new HashSet<>(); + private final Set indexedFields = new HashSet<>(); + private final Set ignoredFields = new HashSet<>(); private ReentrantReadWriteLock modelLock = new ReentrantReadWriteLock(); private volatile boolean hasModels = false; @@ -111,7 +111,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { alfrescoModelDir.mkdir(); } - + loadPersistedModels(); } @@ -124,23 +124,15 @@ public class ModelTracker extends AbstractTracker implements Tracker } /** - * + * */ private void loadPersistedModels() { - HashMap modelMap = new HashMap(); + HashMap modelMap = new HashMap<>(); if (alfrescoModelDir.exists() && alfrescoModelDir.isDirectory()) { // A filter for XML files - FileFilter filter = new FileFilter() - { - @Override - public boolean accept(File pathname) - { - return pathname.isFile() && pathname.getName().endsWith(".xml"); - } - - }; + FileFilter filter = pathname -> pathname.isFile() && pathname.getName().endsWith(".xml"); // List XML files File[] files = alfrescoModelDir.listFiles(filter); if (files != null) @@ -289,26 +281,20 @@ public class ModelTracker extends AbstractTracker implements Tracker long start = System.nanoTime(); List modelDiffs = client.getModelsDiff(coreName, this.infoSrv.getAlfrescoModels()); - HashMap modelMap = new HashMap(); + Map modelMap = new HashMap<>(); for (AlfrescoModelDiff modelDiff : modelDiffs) { switch (modelDiff.getType()) { case CHANGED: + case NEW: AlfrescoModel changedModel = client.getModel(coreName, modelDiff.getModelName()); for (M2Namespace namespace : changedModel.getModel().getNamespaces()) { modelMap.put(namespace.getUri(), changedModel.getModel()); } break; - case NEW: - AlfrescoModel newModel = client.getModel(coreName, modelDiff.getModelName()); - for (M2Namespace namespace : newModel.getModel().getNamespaces()) - { - modelMap.put(namespace.getUri(), newModel.getModel()); - } - break; case REMOVED: // We now remove models as we see them - MNT-17627 // Models have to be deleted from disk before we remove them from memory @@ -317,7 +303,7 @@ public class ModelTracker extends AbstractTracker implements Tracker } } - HashSet loadedModels = new HashSet(); + HashSet loadedModels = new HashSet<>(); for (M2Model model : modelMap.values()) { loadModel(modelMap, loadedModels, model); @@ -367,7 +353,7 @@ public class ModelTracker extends AbstractTracker implements Tracker trackerStats.addModelTime(end - start); - if (true == runPostModelLoadInit) + if (runPostModelLoadInit) { for (Object key : props.keySet()) { @@ -450,10 +436,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { expandedQName = expandQNameImpl(qName); } - // else if (AlfrescoSolrDataModel.nonDictionaryFields.get(qName) == null) - // { - // expandedQName = expandQNameImpl(qName); - // } + return QName.createQName(expandedQName); } @@ -495,10 +478,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { expandedQName = expandQNameImpl(qName); } - // else if (AlfrescoSolrDataModel.nonDictionaryFields.get(qName) == null) - // { - // expandedQName = expandQNameImpl(qName); - // } + return expandedQName; } @@ -513,26 +493,21 @@ public class ModelTracker extends AbstractTracker implements Tracker final String prefix = modelName.toPrefixString(this.infoSrv.getNamespaceDAO()).replace(":", ".") + "."; final String postFix = ".xml"; - File[] toDelete = alfrescoModelDir.listFiles(new FileFilter() - { - @Override - public boolean accept(File pathname) + File[] toDelete = alfrescoModelDir.listFiles(pathname -> { + if (pathname.isDirectory()) { return false; } + String name = pathname.getName(); + if (!name.endsWith(postFix)) { return false; } + if (!name.startsWith(prefix)) { return false; } + // check is number between + String checksum = name.substring(prefix.length(), name.length() - postFix.length()); + try { - if (pathname.isDirectory()) { return false; } - String name = pathname.getName(); - if (false == name.endsWith(postFix)) { return false; } - if (false == name.startsWith(prefix)) { return false; } - // check is number between - String checksum = name.substring(prefix.length(), name.length() - postFix.length()); - try - { - Long.parseLong(checksum); - return true; - } - catch (NumberFormatException nfe) - { - return false; - } + Long.parseLong(checksum); + return true; + } + catch (NumberFormatException nfe) + { + return false; } }); @@ -548,7 +523,7 @@ public class ModelTracker extends AbstractTracker implements Tracker private void loadModel(Map modelMap, HashSet loadedModels, M2Model model) { String modelName = model.getName(); - if (loadedModels.contains(modelName) == false) + if (!loadedModels.contains(modelName)) { for (M2Namespace importNamespace : model.getImports()) { From b44c0fd02b222a6a218fb4adbae9552c812edb53 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 22 Aug 2019 12:48:19 +0200 Subject: [PATCH 31/33] [SEARCH-960] Run ModelTracker even when tracking for all cores is disabled --- .../lifecycle/SolrCoreLoadRegistration.java | 89 +++++++++++++------ 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java index ea4820ef0..c7f46f770 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadRegistration.java @@ -64,6 +64,8 @@ public class SolrCoreLoadRegistration { public static void registerForCore(AlfrescoCoreAdminHandler adminHandler, CoreContainer coreContainer, SolrCore core, String coreName) { + + TrackerRegistry trackerRegistry = adminHandler.getTrackerRegistry(); Properties props = new CoreDescriptorDecorator(core.getCoreDescriptor()).getProperties(); //Prepare cores @@ -79,10 +81,23 @@ public class SolrCoreLoadRegistration { props.putAll(srv.getProps()); adminHandler.getInformationServers().put(coreName, srv); + SolrTrackerScheduler scheduler = adminHandler.getScheduler(); + + // Prevents other threads from registering the ModelTracker at the same time + // Create model tracker and load all the persisted models + createModelTracker(coreName, + trackerRegistry, + props, + coreContainer.getSolrHome(), + repositoryClient, + srv, + scheduler); + + log.info("Starting to track " + coreName); if (Boolean.parseBoolean(props.getProperty("enable.alfresco.tracking", "false"))) { - SolrTrackerScheduler scheduler = adminHandler.getScheduler(); + if (trackerRegistry.hasTrackersForCore(coreName)) { log.info("Trackers for " + coreName+ " is already registered, shutting them down."); @@ -91,28 +106,7 @@ public class SolrCoreLoadRegistration { adminHandler.getInformationServers().remove(coreName); } - // Prevents other threads from registering the ModelTracker at the same time - synchronized (SolrCoreLoadRegistration.class) - { - ModelTracker mTracker = trackerRegistry.getModelTracker(); - if (mTracker == null) - { - log.debug("Creating ModelTracker when registering trackers for core " + coreName); - mTracker = new ModelTracker(coreContainer.getSolrHome(), props, repositoryClient, - coreName, srv); - - trackerRegistry.setModelTracker(mTracker); - - log.info("Ensuring first model sync."); - mTracker.ensureFirstModelSync(); - log.info("Done ensuring first model sync."); - - //Scheduling the ModelTracker. - scheduler.schedule(mTracker, coreName, props); - } - } - - List trackers = createTrackers(coreName, trackerRegistry, props, scheduler, repositoryClient, srv); + List trackers = createCoreTrackers(coreName, trackerRegistry, props, scheduler, repositoryClient, srv); CommitTracker commitTracker = new CommitTracker(props, repositoryClient, coreName, srv, trackers); trackerRegistry.register(coreName, commitTracker); @@ -120,7 +114,8 @@ public class SolrCoreLoadRegistration { log.info("The Trackers are now scheduled to run"); trackers.add(commitTracker); //Add the commitTracker to the list of scheduled trackers that can be shutdown - core.addCloseHook(new CloseHook() { + core.addCloseHook(new CloseHook() + { @Override public void preClose(SolrCore core) { @@ -148,7 +143,12 @@ public class SolrCoreLoadRegistration { * @param srv * @return A list of trackers */ - private static List createTrackers(String coreName, TrackerRegistry trackerRegistry, Properties props, SolrTrackerScheduler scheduler, SOLRAPIClient repositoryClient, SolrInformationServer srv) { + private static List createCoreTrackers(String coreName, + TrackerRegistry trackerRegistry, + Properties props, + SolrTrackerScheduler scheduler, + SOLRAPIClient repositoryClient, + SolrInformationServer srv) { List trackers = new ArrayList(); AclTracker aclTracker = new AclTracker(props, repositoryClient, coreName, srv); @@ -178,6 +178,45 @@ public class SolrCoreLoadRegistration { return trackers; } + + /** + * Create model tracker and load persisted models. + * + * @param coreName + * @param trackerRegistry + * @param props + * @param solrHome + * @param repositoryClient + * @param srv + * @param scheduler + * @return true if model tracker has been created, false if it already exists. + */ + private synchronized static void createModelTracker(String coreName, + TrackerRegistry trackerRegistry, + Properties props, + String solrHome, + SOLRAPIClient repositoryClient, + SolrInformationServer srv, + SolrTrackerScheduler scheduler) + { + ModelTracker mTracker = trackerRegistry.getModelTracker(); + if (mTracker == null) + { + log.debug("Creating ModelTracker"); + mTracker = new ModelTracker(solrHome, props, repositoryClient, + coreName, srv); + + trackerRegistry.setModelTracker(mTracker); + log.info("Ensuring first model sync."); + mTracker.ensureFirstModelSync(); + log.info("Done ensuring first model sync."); + + //Scheduling the ModelTracker. + scheduler.schedule(mTracker, coreName, props); + } + + } + /** * Shuts down the trackers for a core. * From c206fa8b63a861376621bb024296ada5e8726812 Mon Sep 17 00:00:00 2001 From: eliaporciani Date: Thu, 22 Aug 2019 14:44:36 +0200 Subject: [PATCH 32/33] [SEARCH-960] changed variable name --- .../main/java/org/alfresco/solr/tracker/ModelTracker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index cc0d2577f..ba4e00fa9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -289,10 +289,10 @@ public class ModelTracker extends AbstractTracker implements Tracker { case CHANGED: case NEW: - AlfrescoModel changedModel = client.getModel(coreName, modelDiff.getModelName()); - for (M2Namespace namespace : changedModel.getModel().getNamespaces()) + AlfrescoModel model = client.getModel(coreName, modelDiff.getModelName()); + for (M2Namespace namespace : model.getModel().getNamespaces()) { - modelMap.put(namespace.getUri(), changedModel.getModel()); + modelMap.put(namespace.getUri(), model.getModel()); } break; case REMOVED: From fed32bdf7320b9aee8374f8a22700772de559cd2 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Mon, 26 Aug 2019 16:37:51 +0200 Subject: [PATCH 33/33] Every D Ranked bug is fixed. --- .../solr/AlfrescoCoreAdminHandler.java | 2 +- .../alfresco/solr/AlfrescoSolrDataModel.java | 3 +- .../alfresco/solr/SolrInformationServer.java | 19 ++++-- .../component/AsyncBuildSuggestComponent.java | 6 +- .../RewriteFacetParametersComponent.java | 2 +- .../solr/component/TempFileWarningLogger.java | 9 ++- .../solr/query/AbstractSolrCachingScorer.java | 2 +- .../solr/query/MimetypeGroupingCollector.java | 6 +- .../alfresco/solr/query/Solr4QueryParser.java | 65 +++++++++---------- .../solr/tracker/DateQuarterRouter.java | 9 ++- .../AlfrescoSolrClusteringComponent.java | 4 +- .../java/org/alfresco/solr/TrackerState.java | 23 +++++-- .../alfresco/solr/client/SOLRAPIClient.java | 21 +++--- .../alfresco/solr/tracker/TrackerStats.java | 21 +++--- 14 files changed, 107 insertions(+), 85 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index 559ec6855..f38eba6b3 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -951,7 +951,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { if (maxNodeId >= midpoint) { - if(density >= 1) + if(density >= 1 || density == 0) { //This is fully dense shard. I'm not sure if it's possible to have more nodes on the shards //then the offset, but if it does happen don't expand. diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index f5a556a6e..c28d7ae8b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -1195,8 +1195,7 @@ public class AlfrescoSolrDataModel implements QueryConstants public void removeModel(QName modelQName) { - // FIXME: this has no effect. The method should be changed (SEARCH-1482) - modelErrors.remove(modelQName); + modelErrors.remove(getM2Model(modelQName).getName()); dictionaryDAO.removeModel(modelQName); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index c4287b22a..838604cef 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -957,9 +957,10 @@ public class SolrInformationServer implements InformationServer SolrIndexSearcher solrIndexSearcher = refCounted.get(); coreSummary.add("Searcher", solrIndexSearcher.getStatistics()); Map infoRegistry = core.getInfoRegistry(); - for (String key : infoRegistry.keySet()) + for (Entry infos : infoRegistry.entrySet()) { - SolrInfoMBean infoMBean = infoRegistry.get(key); + SolrInfoMBean infoMBean = infos.getValue(); + String key = infos.getKey(); if (key.equals("/alfresco")) { // TODO Do we really need to fixStats in solr4? @@ -2117,8 +2118,9 @@ public class SolrInformationServer implements InformationServer static void addPropertiesToDoc(Map properties, boolean isContentIndexedForNode, SolrInputDocument newDoc, SolrInputDocument cachedDoc, boolean transformContentFlag) { - for (QName propertyQName : properties.keySet()) + for (Entry property : properties.entrySet()) { + QName propertyQName = property.getKey(); newDoc.addField(FIELD_PROPERTIES, propertyQName.toString()); newDoc.addField(FIELD_PROPERTIES, propertyQName.getPrefixString()); @@ -3412,10 +3414,15 @@ public class SolrInformationServer implements InformationServer SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet stack) throws AuthenticationException, IOException, JSONException { - if ((skipDescendantDocsForSpecificTypes && typesForSkippingDescendantDocs.contains(parentNodeMetaData.getType())) || - (skipDescendantDocsForSpecificAspects && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects()))) + + // skipDescendantDocsForSpecificAspects is initialised on a synchronised method, so access must be also synchronised + synchronized (this) { - return; + if ((skipDescendantDocsForSpecificTypes && typesForSkippingDescendantDocs.contains(parentNodeMetaData.getType())) || + (skipDescendantDocsForSpecificAspects && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects()))) + { + return; + } } Set childIds = new HashSet<>(); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/AsyncBuildSuggestComponent.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/AsyncBuildSuggestComponent.java index efcb7d38d..1440ec5a5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/AsyncBuildSuggestComponent.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/AsyncBuildSuggestComponent.java @@ -41,6 +41,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -472,8 +473,9 @@ public class AsyncBuildSuggestComponent extends SearchComponent implements SolrC @Override public long ramBytesUsed() { long sizeInBytes = 0; - for (String key : suggesters.keySet()) { - sizeInBytes += suggesters.get(key).get(ASYNC_CACHE_KEY).ramBytesUsed(); + for (Entry suggester : suggesters.entrySet()) + { + sizeInBytes += suggester.getValue().get(ASYNC_CACHE_KEY).ramBytesUsed(); } return sizeInBytes; } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFacetParametersComponent.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFacetParametersComponent.java index f52fe77f0..33de8a9fb 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFacetParametersComponent.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFacetParametersComponent.java @@ -91,7 +91,7 @@ public class RewriteFacetParametersComponent extends SearchComponent String rows = params.get("rows"); if(rows != null && !rows.isEmpty()) { - Integer row = new Integer(rows); + Integer row = Integer.valueOf(rows); // Avoid +1 in SOLR code which produces null:java.lang.NegativeArraySizeException at at org.apache.lucene.util.PriorityQueue.(PriorityQueue.java:56) if(row > 1000000) { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/TempFileWarningLogger.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/TempFileWarningLogger.java index b2b8d07f3..6f885c3e6 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/TempFileWarningLogger.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/component/TempFileWarningLogger.java @@ -19,7 +19,6 @@ package org.alfresco.solr.component; import java.io.IOException; -import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -53,9 +52,9 @@ public class TempFileWarningLogger log.debug("Looking for temp files matching " + glob + " in directory " + dir); } - try(DirectoryStream stream = Files.newDirectoryStream(dir, glob)) + try { - for (Path file : stream) + for (Path file : Files.newDirectoryStream(dir, glob)) { if (log.isDebugEnabled()) { @@ -74,9 +73,9 @@ public class TempFileWarningLogger public void removeFiles() { - try(DirectoryStream stream = Files.newDirectoryStream(dir, glob)) + try { - for (Path file : stream) + for (Path file : Files.newDirectoryStream(dir, glob)) { file.toFile().delete(); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/AbstractSolrCachingScorer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/AbstractSolrCachingScorer.java index b008175f4..157e5d1c9 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/AbstractSolrCachingScorer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/AbstractSolrCachingScorer.java @@ -47,7 +47,7 @@ public abstract class AbstractSolrCachingScorer extends Scorer static { for(int i = 0; i < cache.length; i++) - cache[i] = new Long(i); + cache[i] = Long.valueOf(i); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/MimetypeGroupingCollector.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/MimetypeGroupingCollector.java index 863d79a83..7253121fb 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/MimetypeGroupingCollector.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/MimetypeGroupingCollector.java @@ -20,6 +20,7 @@ package org.alfresco.solr.query; import java.io.IOException; import java.util.HashMap; +import java.util.Map.Entry; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.AlfrescoSolrDataModel.FieldUse; @@ -111,10 +112,9 @@ public class MimetypeGroupingCollector extends DelegatingCollector rb.rsp.add("analytics", analytics); NamedList fieldCounts = new NamedList<>(); analytics.add("mimetype()", fieldCounts); - for(String key : counters.keySet()) + for (Entry counter : counters.entrySet()) { - Counter counter = counters.get(key); - fieldCounts.add(key, counter.get()); + fieldCounts.add(counter.getKey(), counter.getValue().get()); } if(this.delegate instanceof DelegatingCollector) { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java index 5b138c593..c7dc07382 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java @@ -3301,44 +3301,35 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants protected String getToken(String field, String value, AnalysisMode analysisMode) throws ParseException { - try (TokenStream source = getAnalyzer().tokenStream(field, new StringReader(value))) - { - String tokenised = null; + + TokenStream source = getAnalyzer().tokenStream(field, new StringReader(value)); - while (source.incrementToken()) - { - CharTermAttribute cta = source.getAttribute(CharTermAttribute.class); - OffsetAttribute offsetAtt = source.getAttribute(OffsetAttribute.class); - TypeAttribute typeAtt = null; - if (source.hasAttribute(TypeAttribute.class)) - { - typeAtt = source.getAttribute(TypeAttribute.class); - } - PositionIncrementAttribute posIncAtt = null; - if (source.hasAttribute(PositionIncrementAttribute.class)) - { - posIncAtt = source.getAttribute(PositionIncrementAttribute.class); - } - PackedTokenAttributeImpl token = new PackedTokenAttributeImpl(); - token.setEmpty().copyBuffer(cta.buffer(), 0, cta.length()); - token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); - if (typeAtt != null) - { - token.setType(typeAtt.type()); - } - if (posIncAtt != null) - { - token.setPositionIncrement(posIncAtt.getPositionIncrement()); - } - - tokenised = token.toString(); - } - return tokenised; - } catch (IOException e) + CharTermAttribute cta = source.getAttribute(CharTermAttribute.class); + OffsetAttribute offsetAtt = source.getAttribute(OffsetAttribute.class); + TypeAttribute typeAtt = null; + if (source.hasAttribute(TypeAttribute.class)) { - throw new ParseException("IO" + e.getMessage()); + typeAtt = source.getAttribute(TypeAttribute.class); + } + PositionIncrementAttribute posIncAtt = null; + if (source.hasAttribute(PositionIncrementAttribute.class)) + { + posIncAtt = source.getAttribute(PositionIncrementAttribute.class); + } + PackedTokenAttributeImpl token = new PackedTokenAttributeImpl(); + token.setEmpty().copyBuffer(cta.buffer(), 0, cta.length()); + token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); + if (typeAtt != null) + { + token.setType(typeAtt.type()); + } + if (posIncAtt != null) + { + token.setPositionIncrement(posIncAtt.getPositionIncrement()); } + return token.toString(); + } @Override @@ -5472,11 +5463,13 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants } protected BytesRef analyzeMultitermTerm(String field, String part, Analyzer analyzerIn) { + if (analyzerIn == null) analyzerIn = getAnalyzer(); - try (TokenStream source = analyzerIn.tokenStream(field, part)) { - source.reset(); + try + { + TokenStream source = analyzerIn.tokenStream(field, part); TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); if (!source.incrementToken()) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateQuarterRouter.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateQuarterRouter.java index 8a446c880..c3c32ad2e 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateQuarterRouter.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateQuarterRouter.java @@ -55,6 +55,13 @@ public class DateQuarterRouter implements DocRouter calendar.setTime(date); int month = calendar.get(Calendar.MONTH); int year = calendar.get(Calendar.YEAR); - return Math.ceil(((year * 12) + (month+1)) / 3) % numShards == shardInstance; + + // Avoid using Math.ceil with Integer + int countMonths = ((year * 12) + (month+1)); + int grouping = 3; + int ceilGroupInstance = countMonths / grouping + ((countMonths % grouping == 0) ? 0 : 1); + + return ceilGroupInstance % numShards == shardInstance; + } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrClusteringComponent.java b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrClusteringComponent.java index 62a3f47c3..417e03d42 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrClusteringComponent.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrClusteringComponent.java @@ -280,7 +280,7 @@ public class AlfrescoSolrClusteringComponent extends SearchComponent implements list.add(doc); if (ids != null) { - ids.put(doc, new Integer(docid)); + ids.put(doc, Integer.valueOf(docid)); } } return list; @@ -356,7 +356,7 @@ public class AlfrescoSolrClusteringComponent extends SearchComponent implements /** * @return Expose for tests. */ - Map getSearchClusteringEngines() { + Map getSearchClusteringEnginesView() { return searchClusteringEnginesView; } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java index d585a0c0b..8efed9b1b 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/TrackerState.java @@ -26,6 +26,8 @@ package org.alfresco.solr; +import java.util.concurrent.atomic.AtomicInteger; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,8 +66,21 @@ public class TrackerState private volatile boolean checkedLastAclTransactionTime = false; private volatile boolean checkedLastTransactionTime = false; - private volatile boolean check = false; - private volatile int trackerCycles; + private volatile boolean check = false; + // Handle Thread Safe operations + private volatile TrackerCyclesInteger trackerCycles; + class TrackerCyclesInteger + { + private AtomicInteger value = new AtomicInteger(0); + private void increase() + { + value.incrementAndGet(); + } + private int getValue() + { + return value.get(); + } + } private long timeToStopIndexing; private long lastGoodChangeSetCommitTimeInIndex; @@ -237,13 +252,13 @@ public class TrackerState public int getTrackerCycles() { - return this.trackerCycles; + return this.trackerCycles.getValue(); } public synchronized void incrementTrackerCycles() { log.debug("incrementTrackerCycles from :" + trackerCycles); - this.trackerCycles++; + this.trackerCycles.increase(); log.debug("incremented TrackerCycles to :" + trackerCycles); } diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java index d1a19a51a..136d5ddde 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/SOLRAPIClient.java @@ -38,6 +38,7 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.alfresco.error.AlfrescoRuntimeException; @@ -741,7 +742,7 @@ public class SOLRAPIClient String localeStr = o.has("locale") && !o.isNull("locale") ? o.getString("locale") : null; Locale locale = (o.has("locale") && !o.isNull("locale") ? deserializer.deserializeValue(Locale.class, localeStr) : null); - Long size = o.has("size") && !o.isNull("size") ? o.getLong("size") : null; + long size = o.has("size") && !o.isNull("size") ? o.getLong("size") : 0; String encoding = o.has("encoding") && !o.isNull("encoding") ? o.getString("encoding") : null; String mimetype = o.has("mimetype") && !o.isNull("mimetype") ? o.getString("mimetype") : null; @@ -1247,17 +1248,13 @@ public class SOLRAPIClient this.namespaceDAO = namespaceDAO; // add all default converters to this converter - // TODO find a better way of doing this - Map, Map, Converter>> converters = DefaultTypeConverter.INSTANCE.getConverters(); - for(Class source : converters.keySet()) - { - Map, Converter> converters1 = converters.get(source); - for(Class dest : converters1.keySet()) - { - Converter converter = converters1.get(dest); - instance.addConverter(source, dest, converter); - } - } + for (Entry, Map, Converter>> source : DefaultTypeConverter.INSTANCE.getConverters().entrySet()) + { + for (Entry, Converter> dest : source.getValue().entrySet()) + { + instance.addConverter((Class) source.getKey(), (Class) dest.getKey(), dest.getValue()); + } + } // dates instance.addConverter(String.class, Date.class, new TypeConverter.Converter() diff --git a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java index 6b5f925bf..93291cad9 100644 --- a/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java +++ b/search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/tracker/TrackerStats.java @@ -29,9 +29,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.concurrent.NotThreadSafe; + import org.alfresco.solr.InformationServerCollectionProvider; import org.alfresco.solr.adapters.ISimpleOrderedMap; import org.alfresco.util.Pair; @@ -283,11 +286,10 @@ public class TrackerStats map.add("StdDev", getStandardDeviation()); if (incdludeDetail) { - for (String key : copies.keySet()) - { - IncrementalStats value = copies.get(key); - map.add(key, value.getNamedList(includeHist, includeValues)); - } + for (Entry copy : copies.entrySet()) + { + map.add(copy.getKey(), copy.getValue().getNamedList(includeHist, includeValues)); + } } return map; @@ -382,6 +384,7 @@ public class TrackerStats } + @NotThreadSafe public static class IncrementalStats { Date start = new Date(); @@ -769,7 +772,7 @@ public class TrackerStats { IncrementalStats copy = new IncrementalStats(this.scale, this.buckets, this.server); copy.start = this.start; - copy.max = this.max; + copy.max = this.getMax(); copy.min = this.min; copy.moments[0] = this.moments[0]; copy.moments[1] = this.moments[1];