Merge branches 'feature/SEARCH-2090' and 'master' of https://git.alfresco.com/search_discovery/insightengine into feature/SEARCH-2090

 Conflicts:
	e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSpellCheckTest.java
	insight-engine/alfresco-insight-engine/src/main/java/org/alfresco/solr/sql/SolrSchema.java
	search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java
	search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java
	search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java
This commit is contained in:
agazzarini
2020-03-09 15:42:29 +01:00
58 changed files with 3080 additions and 438 deletions
@@ -17,6 +17,8 @@ $ tree generators/app/templates/
│   ├── .env
│   ├── docker-compose-ce.yml
│   └── docker-compose-ee.yml
├── empty
│   └── empty
├── images
│   ├── alfresco
│   │   ├── Dockerfile
@@ -181,6 +183,10 @@ Sample configuration is available in [images/share/model/sharding-share-config-c
If *Sharding* is selected, a default `share-config-custom-dev.xml` file with required forms configuration for Sharding custom model will be available in deployment folder. Add your configuration to this file.
## Installing custom addons
The generator will create `amps` and `jars` directories within `alfresco/modules` and `share/modules`. Any amps or jars placed in these directories will be installed in the corresponding container.
## Passing parameters from command line
Default values for options can be specified in the command line, using a `--name=value` pattern. When an options is specified in the command line, the question is not prompted to the user, so you can generate a Docker Compose template with no user interaction.
@@ -267,6 +267,16 @@ module.exports = class extends Generator {
)
}
// Empty addons directories.
['alfresco', 'share'].forEach(container => {
['jars', 'amps'].forEach(addonType => {
this.fs.copy(
this.templatePath('empty/empty'),
this.destinationPath(container + '/modules/' + addonType + '/empty')
);
});
});
// Copy Docker Image for Search applying configuration
this.fs.copyTpl(
this.templatePath(imagesDirectory + '/search'),
@@ -23,7 +23,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr
-Dsolr.baseUrl=/solr
-Dindex.subsystem.name=solr6
-Dshare.host=localhost
-Dalfresco.port=8080
@@ -22,7 +22,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr <% if (sharding == 'true') { %>
-Dsolr.baseUrl=/solr <% if (sharding == 'true') { %>
-Dsolr.useDynamicShardRegistration=true <% } %>
-Dindex.subsystem.name=solr6
-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/
@@ -24,7 +24,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr
-Dsolr.baseUrl=/solr
-Dindex.subsystem.name=solr6
-Dshare.host=localhost
-Dalfresco.port=8080
@@ -22,7 +22,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr <% if (sharding == 'true') { %>
-Dsolr.baseUrl=/solr <% if (sharding == 'true') { %>
-Dsolr.useDynamicShardRegistration=true <% } %>
-Dindex.subsystem.name=solr6
@@ -25,6 +25,13 @@ VOLUME ["${ALF_DATA_DIR}/keystore"]
USER root
# Install modules and addons
RUN mkdir -p $TOMCAT_DIR/amps
COPY modules/amps/* $TOMCAT_DIR/amps/
COPY modules/jars/* $TOMCAT_DIR/webapps/alfresco/WEB-INF/lib/
RUN java -jar $TOMCAT_DIR/alfresco-mmt/alfresco-mmt*.jar install \
$TOMCAT_DIR/amps $TOMCAT_DIR/webapps/alfresco -directory -nobackup -force;
# Default value in "repository.properties" is "dir.keystore=classpath:alfresco/keystore"
RUN if [ "$SOLR_COMMS" == "https" ] ; then \
echo -e "\n\
@@ -8,5 +8,12 @@ RUN sed -i '/Connector port="8080"/a scheme="https" secure="true"' /usr/local/to
sed -i '/Connector port="8080"/a proxyName="localhost" proxyPort="<%=port%>"' /usr/local/tomcat/conf/server.xml
<% } %>
# Install modules
RUN mkdir -p $TOMCAT_DIR/amps
COPY modules/amps/* $TOMCAT_DIR/amps/
COPY modules/jars/* $TOMCAT_DIR/webapps/share/WEB-INF/lib/
RUN java -jar $TOMCAT_DIR/alfresco-mmt/alfresco-mmt*.jar install \
$TOMCAT_DIR/amps $TOMCAT_DIR/webapps/share -directory -nobackup -force;
# Copy custom content forms to deployment folder
COPY model/* $TOMCAT_DIR/shared/classes/alfresco/web-extension/
+4 -5
View File
@@ -11,11 +11,10 @@
<name>Search Analytics E2E Tests</name>
<description>Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share</description>
<properties>
<tas.rest.api.version>1.28</tas.rest.api.version>
<tas.rest.api.version>1.26</tas.rest.api.version>
<tas.rest.api.version>1.35</tas.rest.api.version>
<tas.cmis.api.version>1.13</tas.cmis.api.version>
<tas.utility.version>3.0.19</tas.utility.version>
<rm.version>3.3.0</rm.version>
<tas.utility.version>3.0.20</tas.utility.version>
<rm.version>3.3.0.1</rm.version>
<suiteXmlFile>src/test/resources/SearchSuite.xml</suiteXmlFile>
<test.exclude></test.exclude>
<test.include></test.include>
@@ -120,7 +119,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.10</version>
<version>1.18.12</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -23,7 +23,7 @@ public class TestGroup
// Used for TestRail test annotation
public static final String SEARCH = "search";
public static final String REST_API = "rest-api";
public static final String PREUPGRADE = "pre-upgrade";
public static final String POSTUPGRADE = "post-upgrade";
@@ -31,8 +31,13 @@ public class TestGroup
public static final String ASS_MASTER ="ASS_Master"; // Alfresco search services using master/stand alone mode
public static final String EXPLICIT_SHARDING ="Explicit_Sharding"; // Alfresco search services using sharded environment and explicit routing
public static final String SHARDING ="Sharding"; // Alfresco search services using sharded environment
public static final String ASS_SHARDING = "ASS_Sharding"; // Alfresco Search Services using Sharding
public static final String ASS_SHARDING_DB_ID_RANGE = "ASS_Sharding_DB_ID_RANGE"; // Alfresco Search Services using Sharding with DB_ID_RANGE
public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running
public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n
public static final String ACS_60n = "ACS_60n"; // Alfresco Content Services 6.0 or above
public static final String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above
@@ -22,6 +22,7 @@ import org.alfresco.dataprep.SiteService.Visibility;
import org.alfresco.rest.core.RestProperties;
import org.alfresco.rest.core.RestWrapper;
import org.alfresco.rest.model.RestRequestSpellcheckModel;
import org.alfresco.rest.search.Pagination;
import org.alfresco.rest.search.RestRequestHighlightModel;
import org.alfresco.rest.search.RestRequestQueryModel;
import org.alfresco.rest.search.SearchRequest;
@@ -107,6 +108,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
deployCustomModel("model/music-model.xml");
deployCustomModel("model/finance-model.xml");
deployCustomModel("model/sharding-content-model.xml");
}
@BeforeClass (alwaysRun = true)
@@ -393,10 +395,15 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
return restClient.authenticateUser(user).withSearchAPI().search(searchRequest);
}
protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel)
protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel, Pagination paging)
{
SearchRequest searchRequest = new SearchRequest();
searchRequest.setQuery(queryModel);
if (ofNullable(paging).isPresent())
{
searchRequest.setPaging(paging);
}
return restClient.authenticateUser(user).withSearchAPI().search(searchRequest);
}
@@ -436,7 +443,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
*/
protected SearchResponse testSearchQuery(String query, Integer expectedCount, SearchLanguage queryLanguage)
{
SearchResponse response = performSearch(testUser, query, queryLanguage);
SearchResponse response = performSearch(testUser, query, queryLanguage, getDefaultPagingOptions());
if (ofNullable(expectedCount).isPresent())
{
@@ -455,7 +462,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
*/
protected SearchResponse testSearchQueryOrdered(String query, List<String> expectedNames, SearchLanguage queryLanguage)
{
SearchResponse response = performSearch(testUser, query, queryLanguage);
SearchResponse response = performSearch(testUser, query, queryLanguage, getDefaultPagingOptions());
List<String> names = response.getEntries().stream().map(s -> s.getModel().getName()).collect(Collectors.toList());
@@ -474,7 +481,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
*/
protected SearchResponse testSearchQueryUnordered(String query, Set<String> expectedNames, SearchLanguage queryLanguage)
{
SearchResponse response = performSearch(testUser, query, queryLanguage);
SearchResponse response = performSearch(testUser, query, queryLanguage, getDefaultPagingOptions());
Set<String> names = response.getEntries().stream().map(s -> s.getModel().getName()).collect(Collectors.toSet());
@@ -483,7 +490,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
return response;
}
private SearchResponse performSearch(UserModel asUser, String query, SearchLanguage queryLanguage)
protected SearchResponse performSearch(UserModel asUser, String query, SearchLanguage queryLanguage, Pagination paging)
{
RestRequestQueryModel queryModel = new RestRequestQueryModel();
queryModel.setQuery(query);
@@ -498,11 +505,49 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
queryModel.setLanguage(queryLanguage.toString());
}
SearchResponse response = queryAsUser(asUser, queryModel);
SearchResponse response = queryAsUser(asUser, queryModel, paging);
return response;
}
/**
* Returns pagination object with alfresco default settings
* Sets skipCount = 0, maxItems = 100
*
* @return
*/
private Pagination getDefaultPagingOptions()
{
Pagination paging = new Pagination();
paging.setSkipCount(0);
paging.setMaxItems(100);
return paging;
}
/**
* Set the pagination options for the API query
* @param skipCount Integer
* @param maxItems Integer
* @return
*/
protected Pagination setPaging(Integer skipCount, Integer maxItems)
{
Pagination paging = new Pagination();
if (ofNullable(skipCount).isPresent())
{
paging.setSkipCount(skipCount);
}
if (ofNullable(maxItems).isPresent())
{
paging.setMaxItems(maxItems);
}
return paging;
}
/**
* Method to create and run a simple spellcheck query
* When a spellcheck query is run a user, the query inputed and the user query is inputted
@@ -0,0 +1,222 @@
/*
* Copyright (C) 2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.test.search.functional.searchServices.search;
import javax.json.Json;
import javax.json.JsonObject;
import org.alfresco.dataprep.SiteService.Visibility;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.alfresco.utility.constants.UserRole;
import org.alfresco.utility.data.RandomData;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.SiteModel;
import org.alfresco.utility.model.UserModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* Search end point Public API test with Permission checks
*
* @author Meenal Bhave
*/
public class SearchPermissionsTest extends AbstractE2EFunctionalTest
{
private FileModel file1, file2;
private FolderModel parentFolder, folder1, folder2;
private UserModel testUser1, testUser2, testUser3;
@BeforeClass(alwaysRun = true)
public void dataPreparation() throws Exception
{
/*
* Create the following file structure in the same Site : In addition to the preconditions created in dataPreparation
* |- permGrandParent
* |-- permChild1
* |------ permFile1
* |-- permChild2
* |------ permFile2
* |-- permChild3 (Later: In test 2)
*/
parentFolder = new FolderModel("permGrandParent");
dataContent.usingUser(testUser).usingSite(testSite).createFolder(parentFolder);
folder1 = dataContent.usingUser(testUser).usingSite(testSite).usingResource(parentFolder).createFolderCmisApi("permChild1");
folder2 = dataContent.usingUser(testUser).usingSite(testSite).usingResource(parentFolder).createFolderCmisApi("permChild2");
file1 = new FileModel("permFile1", FileType.TEXT_PLAIN, "File1 with inherited permissions");
file2 = new FileModel("permFile2", FileType.TEXT_PLAIN, "File2 with inherited permissions");
// Create test users
testUser1 = dataUser.createRandomTestUser("UserSiteMember");
testUser2 = dataUser.createRandomTestUser("UserNotASiteMemeber");
testUser3 = dataUser.createRandomTestUser("UserWithoutContentAccess");
dataUser.addUserToSite(testUser1, testSite, UserRole.SiteCollaborator);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder1).createContent(file1);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder2).createContent(file2);
// Deny testUser1 - access to file1
JsonObject userPermission = Json
.createObjectBuilder()
.add("permissions",
Json.createObjectBuilder()
.add("isInheritanceEnabled", false)
.add("locallySet",
Json.createObjectBuilder()
.add("authorityId", testUser1.getUsername())
.add("name", "SiteCollaborator")
.add("accessStatus", "DENIED")
)).build();
String putBody = userPermission.toString();
restClient.authenticateUser(testUser).withCoreAPI().usingNode(file1).updateNode(putBody);
// Allow testUser2 - access to file2, user is not a site member
userPermission = Json
.createObjectBuilder()
.add("permissions",
Json.createObjectBuilder()
.add("isInheritanceEnabled", false)
.add("locallySet",
Json.createObjectBuilder()
.add("authorityId", testUser2.getUsername())
.add("name", "SiteContributor")
.add("accessStatus", "ALLOWED")
)).build();
putBody = userPermission.toString();
restClient.authenticateUser(testUser).withCoreAPI().usingNode(file2).updateNode(putBody);
waitForContentIndexing(file2.getContent(), true);
}
@Test(priority = 1)
public void searchResultsRespectInheritedPermissions()
{
/**
* Private Site Folder Structure available for this test
* |- permGrandParent
* |-- permChild1
* |------ permFile1 (inheritance disabled, deny permission to testUser1)
* |-- permChild2
* |------ permFile2 (inheritance disabled, allow permission to testUser2)
*/
// Search as testUser: expect all: 5 results: When user is a Site Manager
SearchResponse response = queryAsUser(testUser, "cm:name:perm*");
int resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 5, "Unexpected Result count for testUser: Expected 5, received: " + resultCount);
// Search as testUser1: expect 3 results: when user is a site member but without permission to a content
response = queryAsUser(testUser1, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 3, "Unexpected Result count for testUser1: Expected 3, received: " + resultCount);
// Search as testUser2: expect 1 result: When user isn't a site member but has granular permissions to a content
response = queryAsUser(testUser2, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 1, "Unexpected Result count for testUser2: Expected 1, received: " + resultCount);
// Search as testUser3: expect none: 0 results: When user isn't a site member / does not have granular permissions to the content
response = queryAsUser(testUser3, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 0, "Unexpected Result count for testUser3: Expected 0, received: " + resultCount);
}
@Test(priority = 2)
public void searchResultsRespectInheritedPermissionsDisabled() throws Exception
{
// Create folder
FolderModel folder3 = dataContent.usingUser(testUser).usingSite(testSite).usingResource(parentFolder).createFolderCmisApi("permChild3");
// Turn off inherited permissions for folder3
JsonObject userPermission = Json.createObjectBuilder().add("permissions", Json.createObjectBuilder().add("isInheritanceEnabled", false)).build();
String putBody = userPermission.toString();
restClient.authenticateUser(testUser).withCoreAPI().usingNode(folder3).updateNode(putBody);
// Wait for indexing
waitForIndexing(folder3.getName(), true);
/**
* Private Site Folder Structure available now for this test
* |- permGrandParent
* |-- permChild1
* |------ permFile1 (inheritance disabled, deny permission to testUser1)
* |-- permChild2
* |------ permFile2 (inheritance disabled, allow permission to testUser2)
* |-- permChild3 (inheritance disabled)
*/
// Search as testUser: expect all: 6 results: When user is a Site Manager
SearchResponse response = queryAsUser(testUser, "cm:name:perm*");
int resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 6, "Unexpected Result count for testUser: Expected 6, received: " + resultCount);
// Search as testUser1: expect 3 results: when user is a site member but without permission to a content
response = queryAsUser(testUser1, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 3, "Unexpected Result count for testUser1: Expected 3, received: " + resultCount);
// Search as testUser2: expect 1 result: When user isn't a site member but has granular permissions to a content
response = queryAsUser(testUser2, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 1, "Unexpected Result count for testUser2: Expected 1, received: " + resultCount);
// Search as testUser3: expect none: 0 results: When user isn't a site member / does not have granular permissions to the content
response = queryAsUser(testUser3, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 0, "Unexpected Result count for testUser3: Expected 0, received: " + resultCount);
}
@Test(priority = 3)
public void searchResultsOnChangingSiteVisibility() throws Exception
{
// Create Site
SiteModel testPermissionsSite = new SiteModel(RandomData.getRandomName("SiteSearchPermissions"));
testPermissionsSite.setVisibility(Visibility.PUBLIC);
dataSite.usingUser(adminUserModel).createSite(testPermissionsSite);
// Add Users to the site
dataUser.addUserToSite(testUser, testPermissionsSite, UserRole.SiteCollaborator);
// Create a folder
String folderName = "Folder" + unique_searchString;
FolderModel folder = dataContent.usingUser(testUser).usingSite(testPermissionsSite).createFolderCmisApi(folderName);
// Query
Assert.assertTrue(waitForIndexing(folder.getName(), true), "Folder isn't yet indexed");
// Edit Site Visibility
testPermissionsSite.setDescription("PrivateSite".concat(testPermissionsSite.getDescription()));
testPermissionsSite.setVisibility(Visibility.PRIVATE);
restClient.authenticateUser(adminUserModel).withCoreAPI().usingSite(testPermissionsSite).updateSite(testPermissionsSite);
// Wait for indexing
Assert.assertTrue(waitForIndexing("description:" + testPermissionsSite.getDescription(), true), "New Site Description isn't yet been indexed");
// Query
SearchResponse response = queryAsUser(testUser, folderName);
int resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 1, "Unexpected Result count for testUser: Expected 1, received: " + resultCount);
}
}
@@ -0,0 +1,258 @@
/*
* Copyright 2019 Alfresco Software, Ltd. All rights reserved.
* License rights for this program may be obtained from Alfresco Software, Ltd.
* pursuant to a written agreement and any use of this program without such an
* agreement is prohibited.
*/
package org.alfresco.test.search.functional.searchServices.search;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.test.search.functional.searchServices.cmis.AbstractCmisE2ETest;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.UserModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* The purpose of this test is to test search query pagination using cmis and afts query
*
* @author Meenal Bhave
*/
public class SearchQueryPaginationTest extends AbstractCmisE2ETest
{
private UserModel testUser2;
private FolderModel testFolder ;
private FileModel testFile;
@BeforeClass(alwaysRun = true)
public void dataPreparation() throws Exception
{
// Create testUser2: This user does not have access to the testSite
testUser2 = dataUser.createRandomTestUser("testUser2");
// Create a new folder and 10 files inside the folder
testFolder = dataContent.usingUser(testUser).usingSite(testSite).createFolder();
for (int i = 0; i < 10; i++)
{
testFile = new FileModel(i + "-File.txt", unique_searchString, "", FileType.TEXT_PLAIN);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(testFolder).createContent(testFile);
}
waitForMetadataIndexing(testFile.getName(), true);
}
@Test(priority = 1)
public void testCmisSearchWithPagination()
{
// Search for the cmis documents using cmis query
String query = "select * from cmis:document";
// Set skipCount = 0, maxItems = 1000
SearchResponse response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 1000));
// Get getTotalItems, Expect hasModeItems = false
Integer totalDocuments = response.getPagination().getTotalItems();
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocument, maxItems = 1000
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, 1000));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = totalDocuments/2
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, totalDocuments/2));
// Check getTotalItems is the same, hasMoreItems = True
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = totalDocuments-2
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(1, totalDocuments-2));
// Check getTotalItems is the same, hasMoreItems = True
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocuments, maxItems = totalDocuments/2
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, totalDocuments/2));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = totalDocuments
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, totalDocuments));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocuments/2+1, maxItems = totalDocuments
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments/2+1, totalDocuments));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocuments, maxItems = totalDocuments
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, totalDocuments));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = 0
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 0));
Assert.assertTrue(response.isEmpty(), "Empty Response, Error is expected when maxItems is <= 0");
// Set skipCount = totalDocuments, maxItems = 0
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, 0));
Assert.assertTrue(response.isEmpty(), "Empty Response, Error is expected when maxItems <= 0");
// Set skipCount = -1, maxItems = 1
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(-1, 1));
Assert.assertTrue(response.isEmpty(), "Empty Response, Error is expected when skipCount < 0");
}
@Test(priority = 2)
public void testPagination()
{
// Search for the files under the testFolder using cmis query
String parentId = testFolder.getNodeRefWithoutVersion();
String query = "select * from cmis:document where IN_FOLDER('" + parentId + "')";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 100));
// Check getTotalItems = 10, Expect hasModeItems = false
testPaginationDetails(response, 10, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = 10: Expect hasModeItems = false
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 10));
testPaginationDetails(response, 10, 0, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = 5: Expect hasModeItems = true
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 5));
testPaginationDetails(response, 10, 0, 5);
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 2, maxItems = 10: Expect hasModeItems = false
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(2, 10));
testPaginationDetails(response, 10, 2, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 2, maxItems = 7: Expect hasModeItems = true
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(2, 7));
testPaginationDetails(response, 10, 2, 7);
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 2, maxItems = 8: Expect hasModeItems = false
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(2, 8));
testPaginationDetails(response, 10, 2, 8);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
@Test(priority = 3)
public void testPaginationRespectsACLs()
{
// Search for the files under the testFolder using cmis query
String parentId = testFolder.getNodeRefWithoutVersion();
String query = "select * from cmis:document where IN_FOLDER('" + parentId + "')";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser2, query, SearchLanguage.CMIS, setPaging(0, 100));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = 1
response = performSearch(testUser2, query, SearchLanguage.CMIS, setPaging(1, 1));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 1, 1);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
@Test(priority = 4)
public void testSearchApiPagination()
{
// Search for the files with specific title
String query = "cm:title:'" + unique_searchString + "'";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(0, 100));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 10, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = 1
response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(1, 1));
// Get getTotalItems, Expect hasModeItems = true
testPaginationDetails(response, 10, 1, 1);
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 9, maxItems = 10
response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(9, 10));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 10, 9, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 10, maxItems = 10
response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(10, 10));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 10, 10, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
@Test(priority = 5)
public void testSearchApiPaginationRespectsACLs()
{
// Search for the files with specific title
String query = "cm:title:'" + unique_searchString + "'";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser2, query, SearchLanguage.AFTS, setPaging(0, 100));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = 1
response = performSearch(testUser2, query, SearchLanguage.AFTS, setPaging(1, 1));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 1, 1);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
private void testPaginationDetails(SearchResponse response, int expectedTotalCount, int skipCount, int maxItems)
{
Assert.assertEquals(response.getPagination().getTotalItems().intValue(), expectedTotalCount, "Unexpected document count");
Assert.assertEquals(response.getPagination().getSkipCount(), skipCount, "Unexpected skip count returned");
Assert.assertEquals(response.getPagination().getMaxItems(), maxItems, "Unexpected maxItems returned");
// count = (total-skipCount) < maxItems ? total-skipCount: maxItems
int expectedCount = (expectedTotalCount < skipCount) ? 0 : (expectedTotalCount-skipCount) < maxItems ? expectedTotalCount-skipCount : maxItems;
Assert.assertEquals(response.getPagination().getCount(), expectedCount, "Unexpected document count");
}
}
@@ -317,8 +317,11 @@ public class SearchSpellCheckTest extends AbstractSearchServicesE2ETest
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "eklipse");
// Add Solr Query, to check the suggestions on each shard
restClient.authenticateUser(testUser).withParams("spellcheck.q=eclipsess&spellcheck=on").withSolrAPI().getSelectQuery();
// Incorrect spelling with no field for file2
response = SearchSpellcheckQuery(testUser, "eclipses", "eclipses");
response = SearchSpellcheckQuery(testUser, "eclipsess", "eclipsess");
// Matching Result, Spellcheck = searchInsteadFor: eklipses
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file.getName());
@@ -463,9 +466,12 @@ public class SearchSpellCheckTest extends AbstractSearchServicesE2ETest
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file2.getName());
testSearchSpellcheckResponse(response, null, null);
// Add Solr Query, to check the suggestions on the shard
restClient.authenticateUser(testUser).withParams("spellcheck.q=spacebur&spellcheck=on").withSolrAPI().getSelectQuery();
// Checks for User 2
// Incorrect spelling for files created no field
response = SearchSpellcheckQuery(testUser2, "superber", "superber");
response = SearchSpellcheckQuery(testUser2, "superbur", "superbur");
// Matching Result, Spellcheck = searchInsteadFor: supercar
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file2.getName());
@@ -20,9 +20,13 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Sets;
import org.alfresco.rest.search.RestInstanceModel;
import org.alfresco.rest.search.RestShardInfoModel;
@@ -58,7 +62,9 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
RestShardInfoModel model = shardInfoModel.getModel();
assertEquals(model.getTemplate(), "rerank");
assertEquals(model.getMode(), "MASTER");
assertEquals(model.getShardMethod(), "DB_ID");
List<String> shardingMethods = Arrays.asList("DB_ID", "DB_ID_RANGE", "EXPLICIT_ID", "ACL_ID", "MOD_ACL_ID", "DATE", "PROPERTY");
String shardingMethod = model.getShardMethod();
assertTrue(shardingMethods.contains(shardingMethod), "Unexpected Sharding Method Found: " + shardingMethod);
assertTrue(model.getHasContent());
assertTrue(stores.contains(model.getStores()));
@@ -75,7 +81,7 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
assertTrue(baseUrls.contains(instance.getBaseUrl()));
// TODO: Ideally Solr Host and Port should be Parameterised
assertEquals(instance.getHost(), "search");
assertNotNull(instance.getHost(), "The solr host is not present");
assertEquals(instance.getPort().intValue(), 8983);
assertEquals(instance.getState(), "ACTIVE");
assertEquals(instance.getMode(), "MASTER");
@@ -127,4 +133,80 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
restClient.authenticateUser(dataUser.createRandomTestUser()).withShardInfoAPI().getInfo();
restClient.assertStatusCodeIs(HttpStatus.INTERNAL_SERVER_ERROR);
}
/**
* This is a test to check that sharding is correctly working on bamboo and locally.
* Include test group 'sharding' on bamboo to enable this test to run
* @throws JsonProcessingException
*/
@Test(groups = { TestGroup.ACS_60n, TestGroup.SHARDING })
public void getShardInfoWith2OrMoreShards() throws JsonProcessingException
{
RestShardInfoModelCollection info = restClient.authenticateUser(dataUser.getAdminUser()).withShardInfoAPI().getInfo();
restClient.assertStatusCodeIs(HttpStatus.OK);
info.assertThat().entriesListIsNotEmpty();
assertEquals(info.getPagination().getTotalItems().intValue(), 2, "Pagination is: " + info.getPagination().getTotalItems().intValue() + " not expected value of 2");
Set<String> stores = Sets.newHashSet("workspace://SpacesStore", "archive://SpacesStore");
List<String> baseUrls = Arrays.asList("/solr/alfresco", "/solr/archive");
List<RestShardInfoModel> entries = info.getEntries();
Set<String> actualStores = entries.stream().map(shardInfoModel -> shardInfoModel.getModel().getStores()).collect(Collectors.toSet());
assertEquals(actualStores, stores, "The number of stores do not match the expected number of stores");
for (RestShardInfoModel shardInfoModel : entries)
{
RestShardInfoModel model = shardInfoModel.getModel();
assertEquals(model.getTemplate(), "rerank", "Template is not rerank, template found: "+ model.getTemplate());
assertEquals(model.getMode(), "MASTER", "Mode is not MASTER, mode found: "+ model.getMode());
assertTrue(model.getHasContent(), "There is no content on the shards");
assertTrue(model.getNumberOfShards()>=2, "Number of shards is not equal to or greater than 2");
List<String> shardingMethods = Arrays.asList("DB_ID", "DB_ID_RANGE", "EXPLICIT_ID", "ACL_ID", "MOD_ACL_ID", "DATE", "PROPERTY");
String shardingMethod = model.getShardMethod();
assertTrue(shardingMethods.contains(shardingMethod), "Unexpected Sharding Method Found: " + shardingMethod);
List<RestShardModel> shards = model.getShards();
assertNotNull(shards, "There are no shards present");
for (RestShardModel shardInstance : shards)
{
List<RestInstanceModel> instanceList = shardInstance.getInstances();
for (RestInstanceModel instanceX : instanceList)
{
assertTrue(baseUrls.contains(instanceX.getBaseUrl()), "The baseUrl is not present, baseUrl found is: " + instanceX.getBaseUrl());
assertEquals(instanceX.getState(), "ACTIVE", "Shard state is not ACTIVE, shard state is: " + instanceX.getState());
assertNotNull(instanceX.getPort(), "There is not port found for the instance");
assertEquals(instanceX.getMode(), "MASTER", "Mode is not MASTER, mode found: "+ instanceX.getMode());
assertTrue(instanceX.getTransactionsRemaining() >= 0, "Transactions remaining is not more than 0, transactions remaining: " + instanceX.getTransactionsRemaining());
String shardParams = (instanceX).getShardParams();
switch (shardingMethod)
{
case "MOD_ACL_ID":
break;
case "ACL_ID":
break;
case "DB_ID":
break;
case "DB_ID_RANGE":
assertTrue(shardParams.contains("shard.range="), "Shard Parameters Not as expected for the Shard Method: DB_ID_RANGE");
break;
case "DATE":
assertTrue(shardParams.contains("shard.key="), "Shard Parameters Not as expected for the Shard Method: DATE");
assertTrue(shardParams.contains("shard.date.grouping="), "Shard Parameters Not as expected for the Shard Method: DATE");
break;
case "PROPERTY":
assertTrue(shardParams.contains("shard.key="), "Shard Parameters Not as expected for the Shard Method: PROPERTY");
assertTrue(shardParams.contains("shard.regex="), "Shard Parameters Not as expected for the Shard Method: PROPERTY");
break;
case "EXPLICIT_ID":
assertTrue(shardParams.contains("shard.key="), "Shard Parameters Not as expected for the Shard Method: EXPLICIT_ID");
break;
default:
throw new AssertionError("Shard Method Not expected: " + model.getShardMethod());
}
}
}
}
}
}
@@ -322,7 +322,9 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest
response.getContext().assertThat().field("facets").isNotEmpty();
RestGenericFacetResponseModel facetResponseModel = response.getContext().getFacets().get(0);
facetResponseModel.assertThat().field("type").is("stats");
facetResponseModel.assertThat().field("label").is(label);
// TODO : Change back to facetResponseModel.assertThat().field("label").is(label);
// when https://issues.alfresco.com/jira/browse/SEARCH-2125 is fixed
facetResponseModel.assertThat().field("label").contains(label);
RestGenericBucketModel bucket = facetResponseModel.getBuckets().get(0);
List<RestGenericMetricModel> metrics = bucket.getMetrics();
assertEquals(metrics.size(),metricsCount);
@@ -0,0 +1,973 @@
/*
* Copyright (C) 2020 Alfresco Software Limited.
* This file is part of Alfresco
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.test.search.functional.searchServices.solr.admin;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.alfresco.rest.core.RestResponse;
import org.alfresco.search.TestGroup;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.springframework.context.annotation.Configuration;
import org.testng.Assert;
import org.testng.annotations.Test;
/**
* End to end tests for SOLR Admin actions REST API, available from:
*
* http://<server>:<port>/solr/admin/cores?action=(actionName)
*
* @author aborroy
*
*/
@Configuration
public class SolrE2eAdminTest extends AbstractE2EFunctionalTest
{
// SOLR default response status codes (returned in responseHeader.status)
private static final Integer SOLR_RESPONSE_STATUS_OK = 0;
// Alfresco SOLR action response status identifiers
private static final String ACTION_RESPONSE_REPORT = "report";
// Default Alfresco SOLR Core Names
private static final List<String> DEFAULT_CORE_NAMES = new ArrayList<>(List.of("alfresco", "archive"));
/**
* Check that SOLR Response Header contains a Query Time (qtime) and status equals to 0
* @param response SOLR REST API Response in JSON
*/
private void checkResponseStatusOk(RestResponse response)
{
Integer qtime = response.getResponse().body().jsonPath().get("responseHeader.QTime");
Assert.assertTrue(qtime >= 0, "Expeted responseHeader.QTime to be a positive number");
Integer status = response.getResponse().body().jsonPath().get("responseHeader.status");
Assert.assertEquals(status, SOLR_RESPONSE_STATUS_OK, "Expected " + SOLR_RESPONSE_STATUS_OK + " in responseHeader.status,");
}
/**
* Node Report for every core.
* @throws Exception
*/
@Test(priority = 1)
public void testNodeReport() throws Exception
{
Integer nodeid = 200;
RestResponse response = restClient.withParams("nodeid=" + nodeid).withSolrAdminAPI().getAction("nodeReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportNodeid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID'");
Assert.assertEquals(reportNodeid, nodeid, "Expected " + nodeid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID',");
});
}
/**
* Node Report for an specific core.
* @throws Exception
*/
@Test(priority = 2)
public void testNodeReportCore() throws Exception
{
final Integer nodeid = 200;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("nodeid=" + nodeid, "core=" + core).withSolrAdminAPI().getAction("nodeReport");
checkResponseStatusOk(response);
Integer reportNodeid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID'");
Assert.assertEquals(reportNodeid, nodeid, "Expected " + nodeid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Node Report requires "nodeid" parameter.
* This test will return an error as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 3)
public void testNodeReportError() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("nodeReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No nodeid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* ACL Report for every core.
* @throws Exception
*/
@Test(priority = 4)
public void testAclReport() throws Exception
{
Integer aclid = 1;
RestResponse response = restClient.withParams("aclid=" + aclid).withSolrAdminAPI().getAction("aclReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportAclid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id'");
Assert.assertEquals(reportAclid, aclid, "Expected " + aclid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id',");
});
}
/**
* ACL Report for an specific core.
* @throws Exception
*/
@Test(priority = 5)
public void testAclReportCore() throws Exception
{
final Integer aclid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("aclid=" + aclid, "core=" + core).withSolrAdminAPI().getAction("aclReport");
checkResponseStatusOk(response);
Integer reportAclid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id'");
Assert.assertEquals(reportAclid, aclid, "Expected " + aclid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* ACL Report requires "aclid" parameter.
* This test will fail as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 6)
public void testAclReportError() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("aclReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No aclid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* TX Report for every core.
* @throws Exception
*/
@Test(priority = 7)
public void testTxReport() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("txReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".TXID");
Assert.assertEquals(reportTxid, txid, "Expected " + txid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".TXID,");
});
}
/**
* TX Report for an specific core.
* @throws Exception
*/
@Test(priority = 8)
public void testTxReportCore() throws Exception
{
final Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("coreName=" + core, "txid=" + txid).withSolrAdminAPI().getAction("txReport");
checkResponseStatusOk(response);
Integer reportTxid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".TXID");
Assert.assertEquals(reportTxid, txid, "Expected " + txid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".TXID,");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Transaction report requires "txid" parameter.
* This test will fail as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 9)
public void testTxReportError() throws Exception
{
String coreName = "alfresco";
RestResponse response = restClient.withParams("coreName=" + coreName).withSolrAdminAPI().getAction("txReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No txid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* ACL TX Report for every core.
* @throws Exception
*/
@Test(priority = 10)
public void testAclTxReport() throws Exception
{
Integer acltxid = 1;
RestResponse response = restClient.withParams("acltxid=" + acltxid).withSolrAdminAPI().getAction("aclTxReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportAcltxidCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount");
Assert.assertEquals(reportAcltxidCount, Integer.valueOf(2), "Expected 2 in " + ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount,");
});
}
/**
* ACL TX Report for specific core.
* @throws Exception
*/
@Test(priority = 11)
public void testAclTxReportCore() throws Exception
{
final Integer acltxid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("acltxid=" + acltxid, "core=" + core).withSolrAdminAPI().getAction("aclTxReport");
checkResponseStatusOk(response);
Integer reportAcltxidCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount");
Assert.assertEquals(reportAcltxidCount, Integer.valueOf(2), "Expected 2 in " + ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount,");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* AclTx report requires "acltxid" parameter.
* This test will fail as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 12)
public void testAclTxReportError() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("aclTxReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No acltxid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* Report for every core.
* @throws Exception
*/
@Test(priority = 13)
public void testReport() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction(ACTION_RESPONSE_REPORT);
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in " + ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count',");
});
}
/**
* Report for specific core.
* @throws Exception
*/
@Test(priority = 14)
public void testReportCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("coreName=" + core).withSolrAdminAPI().getAction(ACTION_RESPONSE_REPORT);
checkResponseStatusOk(response);
Integer reportTxCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in " + ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Report using params.
* @throws Exception
*/
@Test(priority = 15)
public void testReportWithParams() throws Exception
{
Long fromTime = 0l;
Long toTime = 0l;
RestResponse response = restClient.withParams("fromTime=" + fromTime, "toTime=" + toTime).withSolrAdminAPI()
.getAction(ACTION_RESPONSE_REPORT);
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count'");
Assert.assertTrue(reportTxCount == 0, "Expecting 0 in " + ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count',");
});
}
/**
* Summary for every core.
* @throws Exception
*/
@Test(priority = 16)
public void testSummary() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("summary");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxCount = response.getResponse().body().jsonPath().get("Summary." + core + ".'Alfresco Transactions in Index'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in Summary." + core + ".'Alfresco Transactions in Index',");
});
}
/**
* Summary for specific core.
* @throws Exception
*/
@Test(priority = 17)
public void testSummaryCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("summary");
checkResponseStatusOk(response);
Integer reportTxCount = response.getResponse().body().jsonPath().get("Summary." + core + ".'Alfresco Transactions in Index'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in Summary." + core + ".'Alfresco Transactions in Index',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Check every core.
* @throws Exception
*/
@Test(priority = 18)
public void testCheck() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("check");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* Check specific core.
* @throws Exception
*/
@Test(priority = 19)
public void testCheckCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("check");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* This action only applies to DB_ID_RANGE Sharding method.
* This test verifies expected result when using another deployment
* @throws Exception
*/
@Test(priority = 20)
public void testRangeCheck() throws Exception
{
String coreName = "alfresco";
RestResponse response = restClient.withParams("coreName=" + coreName).withSolrAdminAPI().getAction("rangeCheck");
checkResponseStatusOk(response);
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion is not allowed when not using Shard DB_ID_RANGE method,");
}
/**
* When using DB_ID_RANGE Sharding method, expand param is including a number of nodes to be extended.
* @throws Exception
*/
@Test(priority = 21, groups = { TestGroup.ASS_SHARDING_DB_ID_RANGE })
public void testRangeCheckSharding() throws Exception
{
String coreName = "alfresco";
RestResponse response = restClient.withParams("coreName=" + coreName).withSolrAdminAPI().getAction("rangeCheck");
checkResponseStatusOk(response);
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is a positive number when using Shard DB_ID_RANGE method,");
}
/**
* This action only applies to DB_ID_RANGE Sharding method.
* This test verifies expected result when using another deployment
* @throws Exception
*/
@Test(priority = 22)
public void testExpand() throws Exception
{
String coreName = "alfresco";
String add = "1000";
RestResponse response = restClient.withParams("coreName=" + coreName, "add=" + add).withSolrAdminAPI().getAction("expand");
checkResponseStatusOk(response);
// This action only applies to DB_ID_RANGE Sharding method
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion is not allowed when not using Shard DB_ID_RANGE method,");
}
/**
* When using DB_ID_RANGE Sharding method, expand param is including a number of nodes extended.
* @throws Exception
*/
@Test(priority = 23, groups = { TestGroup.ASS_SHARDING_DB_ID_RANGE })
public void testExpandSharding() throws Exception
{
String coreName = "alfresco";
String add = "1000";
RestResponse response = restClient.withParams("coreName=" + coreName, "add=" + add).withSolrAdminAPI().getAction("expand");
checkResponseStatusOk(response);
// This action only applies to DB_ID_RANGE Sharding method
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is a positive number when using Shard DB_ID_RANGE method,");
}
/**
* Purge TX in every core.
* @throws Exception
*/
@Test(priority = 24)
public void testPurge() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("purge");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* Purge TX in specific core.
* @throws Exception
*/
@Test(priority = 25)
public void testPurgeCore() throws Exception
{
final Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core, "txid=" + txid).withSolrAdminAPI().getAction("purge");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Purge with no params produces an empty response.
* @throws Exception
*/
@Test(priority = 26)
public void testPurgeEmpty() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("purge");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* FIX for every core.
* @throws Exception
*/
@Test(priority = 27, dependsOnMethods = "testPurge")
public void testFix() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("fix");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
List<String> txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex");
Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,");
List<String> aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex");
Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,");
});
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* FIX for specific core.
* @throws Exception
*/
@Test(priority = 28)
public void testFixCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("fix");
checkResponseStatusOk(response);
List<String> txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex");
Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,");
List<String> aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex");
Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,");
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* REINDEX for every core.
* @throws Exception
*/
@Test(priority = 29)
public void testReindex() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("reindex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* REINDEX for specific core.
* @throws Exception
*/
@Test(priority = 30)
public void testReindexCore() throws Exception
{
Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core, "txid=" + txid).withSolrAdminAPI().getAction("reindex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* RETRY for every core.
* @throws Exception
*/
@Test(priority = 31)
public void testRetry() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("retry");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
DEFAULT_CORE_NAMES.forEach(core -> {
List<String> errorNodeList = response.getResponse().body().jsonPath().get("action." + core);
Assert.assertEquals(errorNodeList, Arrays.asList(), "Expected no error nodes,");
});
}
/**
* RETRY for specific core.
* @throws Exception
*/
@Test(priority = 32)
public void testRetryCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("retry");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
List<String> errorNodeList = response.getResponse().body().jsonPath().get("action." + core);
Assert.assertEquals(errorNodeList, Arrays.asList(), "Expected no error nodes,");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* INDEX for every core.
* @throws Exception
*/
@Test(priority = 33)
public void testIndex() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("index");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* INDEX for specific core.
* @throws Exception
*/
@Test(priority = 34)
public void testIndexCore() throws Exception
{
final Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core, "txid=" + txid).withSolrAdminAPI().getAction("index");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Reloads default log4j properties into memory.
* @throws Exception
*/
@Test(priority = 35)
public void testLog4J() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("log4j");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* This REST API call will fail as the specified resource to reload doesn't exist.
* @throws Exception
*/
@Test(priority = 36)
public void testLog4JError() throws Exception
{
RestResponse response = restClient.withParams("resource=log4j-unexisting.properties").withSolrAdminAPI().getAction("log4j");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* This test will fail if it's executed twice
* @throws Exception
*/
@Test(priority = 37)
public void testNewCore() throws Exception
{
String core = "newCore";
String storeRef = "workspace://SpacesStore";
String template = "rerank";
RestResponse response = restClient.withParams("coreName=" + core, "storeRef=" + storeRef, "template=" + template)
.withSolrAdminAPI().getAction("newCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
String actionCore = response.getResponse().body().jsonPath().get("action.core");
Assert.assertEquals(actionCore, core, "Created core name is expected in action.core,");
}
/**
* When creating a core that already exists, this action fails.
* @throws Exception
*/
@Test(priority = 38)
public void testNewCoreError() throws Exception
{
String core = "alfresco";
String template = "rerank";
RestResponse response = restClient.withParams("coreName=" + core, "template=" + template)
.withSolrAdminAPI().getAction("newCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* Reloads core configuration in memory.
* @throws Exception
*/
@Test(priority = 39)
public void testUpdateCore() throws Exception
{
String core = "alfresco";
RestResponse response = restClient.withParams("coreName=" + core).withSolrAdminAPI().getAction("updateCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* When updating a core that doesn't exist, this action fails.
* @throws Exception
*/
@Test(priority = 40)
public void testUpdateCoreError() throws Exception
{
String core = "nonExistingCore";
RestResponse response = restClient.withParams("coreName=" + core).withSolrAdminAPI().getAction("updateCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* This test updates "shared.properties" memory loading for every SOLR core.
* @throws Exception
*/
@Test(priority = 41)
public void testUpdateShared() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("updateShared");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* This test will fail if it's executed twice
* @throws Exception
*/
@Test(priority = 42)
public void testNewDefaultCore() throws Exception
{
String core = "newDefaultCore";
String storeRef = "workspace://SpacesStore";
String template = "rerank";
RestResponse response = restClient
.withParams("coreName=" + core, "storeRef=" + storeRef, "template=" + template)
.withSolrAdminAPI().getAction("newDefaultIndex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
String actionCore = response.getResponse().body().jsonPath().get("action.core");
Assert.assertEquals(actionCore, core, "Created core name is expected in action.core,");
}
/**
* When creating a core that already exists, this action fails.
* @throws Exception
*/
@Test(priority = 43)
public void testNewDefaultCoreError() throws Exception
{
String core = "alfresco";
String template = "rerank";
RestResponse response = restClient.withParams("coreName=" + core, "template=" + template)
.withSolrAdminAPI().getAction("newDefaultIndex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* This test has to be executed after "testNewCore" test, otherwise it will fail
*/
@Test(priority = 99, dependsOnMethods = ("testNewCore"))
public void testRemoveCore() throws Exception
{
String core = "newCore";
String storeRef = "workspace://SpacesStore";
RestResponse response = restClient
.withParams("coreName=" + core, "storeRef=" + storeRef)
.withSolrAdminAPI().getAction("removeCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
}
@@ -20,7 +20,21 @@
<exclude name="org.alfresco.test.search.functional.searchServices.search.rm"/>
</package>
</packages>
<!-- Despite this class is included in Search Services package, needs to be excluded in order to be executed as the last one -->
<classes>
<class name="org.alfresco.test.search.functional.searchServices.solr.admin.SolrE2eAdminTest">
<methods>
<exclude name=".*" />
</methods>
</class>
</classes>
</test>
<!-- This is deliberately scheduled at the end of the test suite because it messes with the cores and might break other tests -->
<test name="Admin">
<classes>
<class name="org.alfresco.test.search.functional.searchServices.solr.admin.SolrE2eAdminTest" />
</classes>
</test>
</suite>
+13
View File
@@ -433,11 +433,24 @@ cd packaging/target/docker-resources/6.x
docker-compose up
```
##Docker Master-Slave setup
We have seperate docker compose file for slave. To setup Master slave setup
`docker-compose -f docker-compose.yml -f ./master-slave/docker-compose.slave.yml up`
The slave running behind the nginx load balancer under 8084, so we can spin up multiple slaves with the same port. To deploy multiple slaves
`docker-compose -f docker-compose.yml -f ./master-slave/docker-compose.slave.yml up --scale search_slave=2`
This will start up Alfresco, Postgres, Share and SearchServices. You can access the applications using the following URLs:
* Alfresco: http://localhost:8081/alfresco
* Share: http://localhost:8082/share
* Solr: http://localhost:8083/solr
* Solr-slave: http://localhost:8084/solr
Note: Once deployed and all services up and running, goto <http://localhost:8081/alfresco/s/enterprise/admin/admin-searchservice?m=admin-console.success>. Scroll down and click save button.
This is the bug which currently working on [SEARCH-2085](https://issues.alfresco.com/jira/browse/SEARCH-2085)
If you start version 5.x instead you can also access the API Explorer:
@@ -4,7 +4,7 @@ Date: 09/04/2019
## Status
Approved
~~Approved~~ Postponed, see [version 2](0009-message-driven-content-tracker-next-gen.md)
## Context
@@ -0,0 +1,28 @@
# Message Driven Content Tracker v2
Date: 05/02/2020
## Status
WIP
## Context
This is a second iteration of content tracking via message bus design. See [previous version](0007-message-driven-content-tracker.md).
New content tracker implementation will be based on new Search Services architecture (SS v2.0 or next gen). Main context behind this decision is almost the same as for v1 - get more throughput by leveraging new Transform Service.
## Decision
The decision is based on [version 1](0007-message-driven-content-tracker.md). The main differences are:
* Shared File Store may not be the right option as it is only available for Enterprise. Alternatively the URL to content can point to other locations. (TBC)
* The change in behaviour requires a major release of Search Services, most likely version 2.0.
* The changes in Content Repository will be available from version 6.3.
* The synchronous transformation APIs will remain functional until 7.0.
Details of the architecture to be clarified (WIP).
## Consequences
Additional latency will be introduced due to the extra calls when using the Transform Service. Also the transformation capabilities are much more limited.
This design highlights a major difference in behaviour, which requires a major release of Search Services.
+2 -2
View File
@@ -168,7 +168,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.2.4</version>
<version>3.3.0</version>
<scope>test</scope>
</dependency>
@@ -181,7 +181,7 @@
<dependency>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>randomizedtesting-runner</artifactId>
<version>2.7.6</version>
<version>2.7.7</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -158,7 +158,6 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
@@ -203,7 +202,6 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DelegatingCollector;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.QueryWrapperFilter;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.AddUpdateCommand;
@@ -2097,11 +2095,13 @@ public class SolrInformationServer implements InformationServer
doc.setField(FIELD_ISNODE, "T");
doc.setField(FIELD_TENANT, AlfrescoSolrDataModel.getTenantId(metadata.getTenantDomain()));
updatePathRelatedFields(metadata, doc);
updateNamePathRelatedFields(metadata, doc);
updateAncestorRelatedFields(metadata, doc);
doc.setField(FIELD_PARENT_ASSOC_CRC, metadata.getParentAssocsCrc());
if (cascadeTrackingEnabled())
{
updatePathRelatedFields(metadata, doc);
updateNamePathRelatedFields(metadata, doc);
updateAncestorRelatedFields(metadata, doc);
doc.setField(FIELD_PARENT_ASSOC_CRC, metadata.getParentAssocsCrc());
}
ofNullable(metadata.getOwner()).ifPresent(owner -> doc.setField(FIELD_OWNER, owner));
@@ -2244,7 +2244,7 @@ public class SolrInformationServer implements InformationServer
QName propertyQName = property.getKey();
document.addField(FIELD_PROPERTIES, propertyQName.toString());
document.addField(FIELD_PROPERTIES, propertyQName.getPrefixString());
PropertyValue value = property.getValue();
if(value != null)
{
@@ -3123,7 +3123,8 @@ public class SolrInformationServer implements InformationServer
{
if (!activeTrackerThreads.contains(Thread.currentThread().getId()))
{
throw new TrackerStateException("The trackers work was rolled back by another tracker error");
throw new TrackerStateException(
"The trackers work was rolled back by another tracker error. The original cause has been dumped previously in the log.");
}
}
finally
@@ -3255,7 +3256,6 @@ public class SolrInformationServer implements InformationServer
return doc;
}
private void updatePathRelatedFields(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
clearFields(doc, FIELD_PATH, FIELD_SITE, FIELD_TAG, FIELD_TAG_SUGGEST, FIELD_APATH, FIELD_ANAME);
@@ -3407,12 +3407,13 @@ public class SolrInformationServer implements InformationServer
AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
addDocCmd.overwrite = overwrite;
addDocCmd.solrDoc = document;
updatePathRelatedFields(nodeMetaData, document);
updateNamePathRelatedFields(nodeMetaData, document);
updateAncestorRelatedFields(nodeMetaData, document);
processor.processAdd(addDocCmd);
if (cascadeTrackingEnabled())
{
updatePathRelatedFields(nodeMetaData, document);
updateNamePathRelatedFields(nodeMetaData, document);
updateAncestorRelatedFields(nodeMetaData, document);
}
processor.processAdd(addDocCmd);
}
}
}
@@ -3462,7 +3463,7 @@ public class SolrInformationServer implements InformationServer
SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet<Long> stack)
throws AuthenticationException, IOException, JSONException
{
// skipDescendantDocsForSpecificAspects is initialised on a synchronised method, so access must be also synchronised
// skipDescendantDocsForSpecificAspects is initialised on a synchronised method, so access must be also synchronised
synchronized (this)
{
if ((skipDescendantDocsForSpecificTypes && typesForSkippingDescendantDocs.contains(parentNodeMetaData.getType())) ||
@@ -68,6 +68,10 @@ public abstract class AbstractTracker implements Tracker
protected boolean transformContent;
String shardTemplate;
protected volatile boolean rollback;
/**
* When rollback is set, original error is also gathered in order to provide detailed logging.
*/
protected Throwable rollbackCausedBy;
protected final Type type;
/*
@@ -176,13 +180,13 @@ public abstract class AbstractTracker implements Tracker
assert(assertTrackerStateRemainsNull());
}
LOGGER.info("... Running {} for core [{}]", this.getClass().getSimpleName(), coreName);
LOGGER.info("[CORE {}] Running {}", coreName, this.getClass().getSimpleName());
if(this.state == null)
{
this.state = getTrackerState();
LOGGER.debug("Global Tracker State set to: {}", this.state.toString());
LOGGER.debug("[CORE {}] Global Tracker State set to: {}", coreName, this.state.toString());
this.state.setRunning(true);
}
else
@@ -199,34 +203,29 @@ public abstract class AbstractTracker implements Tracker
}
catch(IndexTrackingShutdownException t)
{
setRollback(true);
LOGGER.info("Stopping index tracking for {} - {}", getClass().getSimpleName(), coreName);
setRollback(true, t);
LOGGER.info("[CORE {}] Stopping index tracking for {}", coreName, getClass().getSimpleName());
}
catch(Throwable t)
{
setRollback(true);
setRollback(true, t);
if (t instanceof SocketTimeoutException || t instanceof ConnectException)
{
LOGGER.warn("[CORE {}] Tracking communication timed out for {}", coreName, getClass().getSimpleName());
if (LOGGER.isDebugEnabled())
{
// DEBUG, so give the whole stack trace
LOGGER.warn("Tracking communication timed out for {} - {}", getClass().getSimpleName(), coreName, t);
}
else
{
// We don't need the stack trace. It timed out.
LOGGER.warn("Tracking communication timed out for for {} - {}", getClass().getSimpleName(), coreName);
LOGGER.debug("[CORE {}] Stack trace", coreName, t);
}
}
else
{
LOGGER.error("Tracking failed for for {} - {}", getClass().getSimpleName(), coreName, t);
LOGGER.error("[CORE {}] Tracking failed for {}", coreName, getClass().getSimpleName(), t);
}
}
}
catch (InterruptedException e)
{
LOGGER.error("Semaphore interrupted for for {} - {}", getClass().getSimpleName(), coreName, e);
LOGGER.error("[CORE {}] Semaphore interrupted for {}", coreName, getClass().getSimpleName(), e);
}
finally
{
@@ -244,10 +243,16 @@ public abstract class AbstractTracker implements Tracker
{
return this.rollback;
}
public Throwable getRollbackCausedBy()
{
return this.rollbackCausedBy;
}
public void setRollback(boolean rollback)
public void setRollback(boolean rollback, Throwable rollbackCausedBy)
{
this.rollback = rollback;
this.rollbackCausedBy = rollbackCausedBy;
}
private void continueState()
@@ -40,6 +40,7 @@ abstract class AbstractWorkerRunnable implements Runnable
public void run()
{
boolean failed = true;
Exception failCausedBy = null;
try
{
doWork();
@@ -47,7 +48,8 @@ abstract class AbstractWorkerRunnable implements Runnable
}
catch (Exception e)
{
log.warn("Index tracking batch hit an unrecoverable error ", e);
log.warn("Index tracking batch hit an unrecoverable error ", e);
failCausedBy = e;
}
finally
{
@@ -55,12 +57,12 @@ abstract class AbstractWorkerRunnable implements Runnable
queueHandler.removeFromQueueAndProdHead(this);
if(failed)
{
onFail();
onFail(failCausedBy);
}
}
}
abstract protected void doWork() throws Exception;
abstract protected void onFail();
abstract protected void onFail(Throwable failCausedBy);
}
@@ -54,7 +54,7 @@ import org.slf4j.LoggerFactory;
public class AclTracker extends AbstractTracker
{
protected final static Logger log = LoggerFactory.getLogger(AclTracker.class);
protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class);
private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 100;
private static final int DEFAULT_ACL_BATCH_SIZE = 10;
@@ -135,8 +135,15 @@ public class AclTracker extends AbstractTracker
indexAcl(readers, false);
}
this.infoSrv.indexAclTransaction(changeSet, false);
LOGGER.info("[CORE {}] - INDEX ACTION - AclChangeSetId {} has been indexed", coreName, aclChangeSetId);
requiresCommit = true;
}
else
{
LOGGER.info(
"[CORE {}] - INDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed",
coreName, aclChangeSetId);
}
}
checkShutdown();
}
@@ -160,6 +167,7 @@ public class AclTracker extends AbstractTracker
//AclReaders r = readers.get(0);
//System.out.println("############## READERS ID:"+r.getId()+":"+r.getReaders());
indexAcl(readers, false);
LOGGER.info("[CORE {}] - INDEX ACTION - AclId {} has been indexed", coreName, aclId);
}
checkShutdown();
}
@@ -187,8 +195,15 @@ public class AclTracker extends AbstractTracker
}
this.infoSrv.indexAclTransaction(changeSet, true);
LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} has been reindexed", coreName, aclChangeSetId);
requiresCommit = true;
}
else
{
LOGGER.info(
"[CORE {}] - REINDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed",
coreName, aclChangeSetId);
}
}
checkShutdown();
}
@@ -212,6 +227,7 @@ public class AclTracker extends AbstractTracker
Acl acl = new Acl(0, aclId);
List<AclReaders> readers = client.getAclReaders(Collections.singletonList(acl));
indexAcl(readers, true);
LOGGER.info("[CORE {}] - REINDEX ACTION - aclId {} has been reindexed", coreName, aclId);
requiresCommit = true;
}
checkShutdown();
@@ -231,6 +247,7 @@ public class AclTracker extends AbstractTracker
if (aclChangeSetId != null)
{
this.infoSrv.deleteByAclChangeSetId(aclChangeSetId);
LOGGER.info("[CORE {}] - PURGE ACTION - Purged aclChangeSetId {}", coreName, aclChangeSetId);
}
checkShutdown();
}
@@ -245,6 +262,7 @@ public class AclTracker extends AbstractTracker
if (aclId != null)
{
this.infoSrv.deleteByAclId(aclId);
LOGGER.info("[CORE {}] - PURGE ACTION - Purged aclId {}", coreName, aclId);
}
checkShutdown();
}
@@ -316,7 +334,7 @@ public class AclTracker extends AbstractTracker
{
state.setCheckedLastAclTransactionTime(true);
state.setCheckedFirstAclTransactionTime(true);
log.info("No acl transactions found - no verification required");
LOGGER.info("[CORE {}] - No acl transactions found - no verification required", coreName);
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
@@ -340,20 +358,20 @@ public class AclTracker extends AbstractTracker
if (setSize == 0)
{
log.error("First acl transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
LOGGER.error("[CORE {}] First acl transaction was not found with the correct timestamp.", coreName);
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial acl transaction not found with correct timestamp");
}
else if (setSize == 1)
{
state.setCheckedFirstTransactionTime(true);
log.info("Verified first acl transaction and timestamp in index");
LOGGER.info("[CORE {}] Verified first acl transaction and timestamp in index", coreName);
}
else
{
log.warn("Duplicate initial acl transaction found with correct timestamp");
LOGGER.warn("[CORE {}] Duplicate initial acl transaction found with correct timestamp", coreName);
}
}
}
@@ -374,19 +392,19 @@ public class AclTracker extends AbstractTracker
AclChangeSet maxAclTxInIndex = this.infoSrv.getMaxAclChangeSetIdAndCommitTimeInIndex();
if (maxAclTxInIndex.getCommitTimeMs() > maxChangeSetCommitTimeInRepo)
{
log.error("Last acl transaction was found in index with timestamp later than that of repository.");
log.error("Max Acl Tx In Index: " + maxAclTxInIndex.getId() + ", In Repo: " + maxChangeSetIdInRepo);
log.error("Max Acl Tx Commit Time In Index: " + maxAclTxInIndex.getCommitTimeMs() + ", In Repo: "
LOGGER.error("[CORE {}] Last acl transaction was found in index with timestamp later than that of repository.", coreName);
LOGGER.error("Max Acl Tx In Index: " + maxAclTxInIndex.getId() + ", In Repo: " + maxChangeSetIdInRepo);
LOGGER.error("Max Acl Tx Commit Time In Index: " + maxAclTxInIndex.getCommitTimeMs() + ", In Repo: "
+ maxChangeSetCommitTimeInRepo);
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Last acl transaction found in index with incorrect timestamp");
}
else
{
state.setCheckedLastAclTransactionTime(true);
log.info("Verified last acl transaction timestamp in index less than or equal to that of repository.");
LOGGER.info("[CORE {}] - Verified last acl transaction timestamp in index less than or equal to that of repository.", coreName);
}
}
}
@@ -416,7 +434,7 @@ public class AclTracker extends AbstractTracker
AclChangeSets aclChangeSets;
// step forward in time until we find something or hit the time bound
// max id unbounded
Long startTime = fromCommitTime == null ? Long.valueOf(0L) :fromCommitTime;
Long startTime = fromCommitTime == null ? Long.valueOf(0L) : fromCommitTime;
do
{
aclChangeSets = client.getAclChangeSets(startTime, null, startTime + actualTimeStep, null, maxResults);
@@ -645,30 +663,58 @@ public class AclTracker extends AbstractTracker
*/
this.state = getTrackerState();
Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex());
aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
state.getTimeToStopIndexing());
setLastChangeSetIdAndCommitTimeInTrackerState(aclChangeSets, state);
log.info("Scanning Acl change sets ...");
if (aclChangeSets.getAclChangeSets().size() > 0) {
log.info(".... from " + aclChangeSets.getAclChangeSets().get(0));
log.info(".... to " + aclChangeSets.getAclChangeSets().get(aclChangeSets.getAclChangeSets().size() - 1));
} else {
log.info(".... none found after lastTxCommitTime " + fromCommitTime);
if (aclChangeSets.getAclChangeSets().size() > 0)
{
LOGGER.info("{}-[CORE {}] Found {} ACL change sets after lastTxCommitTime {}, ACL Change Sets from {} to {}",
Thread.currentThread().getId(),
coreName,
aclChangeSets.getAclChangeSets().size(),
fromCommitTime,
aclChangeSets.getAclChangeSets().get(0),
aclChangeSets.getAclChangeSets().get(aclChangeSets.getAclChangeSets().size() - 1));
}
else
{
LOGGER.info("{}-[CORE {}] No ACL change set found after lastTxCommitTime {}",
Thread.currentThread().getId(), coreName, fromCommitTime);
}
ArrayList<AclChangeSet> changeSetBatch = new ArrayList<AclChangeSet>();
for (AclChangeSet changeSet : aclChangeSets.getAclChangeSets()) {
for (int i = 0; i < aclChangeSets.getAclChangeSets().size(); i++)
{
AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(i);
boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() &&
infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
if (isInIndex) {
if (isInIndex)
{
// Logging progress for large ACL Change Set tracking every 100 tracked ACLs
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Change Set Id was already indexed: {}",
Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId());
}
changeSetsFound.add(changeSet);
} else {
}
else
{
// Logging progress for ACL Change Set
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Current Change Set Id to be indexed: {}",
Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId());
}
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()) {
@@ -731,10 +777,12 @@ public class AclTracker extends AbstractTracker
{
getWriteLock().release();
}
}
while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false));
LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount);
log.info("total number of acls updated: " + totalAclCount);
}
private void setLastChangeSetIdAndCommitTimeInTrackerState(AclChangeSets aclChangeSets, TrackerState state)
@@ -797,12 +845,19 @@ public class AclTracker extends AbstractTracker
ArrayList<Acl> aclBatch = new ArrayList<Acl>();
List<Acl> acls = client.getAcls(nonEmptyChangeSets, null, Integer.MAX_VALUE);
if (LOGGER.isDebugEnabled())
{
LOGGER.debug("{}-[CORE {}] Found {} Acls from Acl Change Sets: {}", Thread.currentThread().getId(),
coreName, acls.size(), nonEmptyChangeSets);
}
for (Acl acl : acls)
{
if (log.isDebugEnabled())
if (LOGGER.isTraceEnabled())
{
log.debug(acl.toString());
LOGGER.trace("{}-[CORE {}] Adding ACL {} to scheduled indexing job", Thread.currentThread().getId(),
coreName, acl.toString());
}
aclBatch.add(acl);
if (aclBatch.size() > aclBatchSize)
@@ -845,9 +900,9 @@ public class AclTracker extends AbstractTracker
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
setRollback(true);
setRollback(true, failCausedBy);
}
private List<Acl> filterAcls(List<Acl> acls)
@@ -114,9 +114,9 @@ public class CascadeTracker extends AbstractTracker implements Tracker
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
setRollback(true);
setRollback(true, failCausedBy);
}
}
@@ -190,6 +190,17 @@ public class CommitTracker extends AbstractTracker
}
infoSrv.rollback();
// Log reasons why the rollback is performed
if (aclTracker.getRollbackCausedBy() != null)
{
log.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy());
}
if (metadataTracker.getRollbackCausedBy() != null)
{
log.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy());
}
}
catch (Exception e)
{
@@ -198,19 +209,19 @@ public class CommitTracker extends AbstractTracker
finally
{
//Reset acl Tracker
aclTracker.setRollback(false);
aclTracker.setRollback(false, null);
aclTracker.invalidateState();
//Reset metadataTracker
metadataTracker.setRollback(false);
metadataTracker.setRollback(false, null);
metadataTracker.invalidateState();
//Reset contentTracker
contentTracker.setRollback(false);
contentTracker.setRollback(false, null);
contentTracker.invalidateState();
//Reset cascadeTracker
cascadeTracker.ifPresent(c -> c.setRollback(false));
cascadeTracker.ifPresent(c -> c.setRollback(false, null));
cascadeTracker.ifPresent(c -> invalidateState());
//Release the locks
@@ -162,9 +162,10 @@ public class ContentTracker extends AbstractTracker implements Tracker
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
// Will redo if not persisted
// This will be redone in future tracking operations
log.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy);
}
}
}
@@ -40,6 +40,7 @@ import org.alfresco.service.cmr.dictionary.PropertyDefinition;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.BoundedDeque;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.NodeReport;
import org.alfresco.solr.TrackerState;
@@ -52,8 +53,8 @@ import java.util.Properties;
/**
* Superclass for all components which are able to inform Alfresco about the hosting node state.
* This has been introduced in SEARCH-1752 for splitting the dual responsibility of the {@link MetadataTracker}.
* As consequence of that, this class contains all the members needed for obtaining a valid
* This has been introduced in SEARCH-1752 for splitting the dual responsibility of the {@link org.alfresco.solr.tracker.MetadataTracker}.
* As consequence of that, this class contains only the members needed for obtaining a valid
* {@link org.alfresco.repo.index.shard.ShardState} that can be periodically communicated to Alfresco.
*
* @author Andrea Gazzarini
@@ -176,15 +177,15 @@ public abstract class CoreStatePublisher extends AbstractTracker
* The {@link ShardState} is primarily used in two places:
*
* <ul>
* <li>Transaction tracking: (see {@link MetadataTracker#trackTransactions()}): for pulling/tracking transactions from Alfresco</li>
* <li>Transaction tracking: (see {@link MetadataTracker#getSomeTransactions(BoundedDeque, Long, long, int, long}): for pulling/tracking transactions from Alfresco</li>
* <li>
* DynamicSharding: when the {@link MetadataTracker} is running on a slave instance it doesn't actually act
* as a tracker, it calls Alfresco to register the state of the node (the shard) without pulling any transactions.
* As consequence of that, Alfresco will be aware about the shard which will be included in subsequent queries.
* DynamicSharding: the {@link MetadataTracker} is not running on a slave instances; in those cases a special
* "tracker" ({@link SlaveCoreStatePublisher}) will be in charge to send the correspondin shard state to Alfresco.
* </li>
* </ul>
*
* @return the {@link ShardState} instance which stores the current state of the hosting shard.
* @see SlaveCoreStatePublisher
*/
ShardState getShardState()
{
@@ -198,10 +199,11 @@ public abstract class CoreStatePublisher extends AbstractTracker
HashMap<String, String> propertyBag = new HashMap<>();
propertyBag.put("coreName", coreName);
HashMap<String, String> extendedPropertyBag = new HashMap<>(propertyBag);
updateShardProperty();
shardProperty.ifPresent(p -> extendedPropertyBag.putAll(docRouter.getProperties(p)));
extendedPropertyBag.putAll(docRouter.getProperties(shardProperty));
return ShardStateBuilder.shardState()
.withMaster(isMaster)
@@ -19,6 +19,7 @@
package org.alfresco.solr.tracker;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -108,9 +109,8 @@ public class DBIDRangeRouter implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return Map.of(DocRouterFactory.SHARD_RANGE_KEY, startRange + "-" + expandableRange);
}
}
@@ -26,10 +26,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Map;
import java.util.Optional;
import static java.util.Collections.emptyMap;
/**
* The date-based sharding assigns dates sequentially through shards based on the month.
@@ -121,14 +123,13 @@ public class DateMonthRouter implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return (shardProperty == null ?
Collections.emptyMap() :
Map.of(DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString(),
DocRouterFactory.SHARD_DATE_GROUPING_KEY, String.valueOf(grouping)));
return shardProperty
.map(QName::getPrefixString)
.map(prefix -> Map.of(
DocRouterFactory.SHARD_KEY_KEY, prefix,
DocRouterFactory.SHARD_DATE_GROUPING_KEY, String.valueOf(grouping)))
.orElse(emptyMap());
}
}
@@ -22,6 +22,7 @@ import org.alfresco.solr.client.Node;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.client.Acl;
@@ -65,13 +66,13 @@ public interface DocRouter
Boolean routeNode(int shardCount, int shardInstance, Node node);
/**
* Get additional properties to "shardProperty" depending on the Shard Method
* @param shardProperty custom property used to configure the Router
* Get additional properties to "shardProperty" depending on the Shard Method.
*
* @param shardProperty custom property used to configure the Router. Note not all routers need that.
* @return pair of key, value
*/
default public Map<String, String> getProperties(QName shardProperty) {
default Map<String, String> getProperties(Optional<QName> shardProperty) {
return Collections.emptyMap();
}
}
@@ -9,6 +9,7 @@ import java.util.Objects;
import static java.util.Optional.ofNullable;
import java.util.Map;
import java.util.Optional;
/**
* A composable {@link DocRouter} which consists of
@@ -47,9 +48,8 @@ public class DocRouterWithFallback implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return primaryStrategy.getProperties(shardProperty);
}
}
@@ -18,13 +18,15 @@
*/
package org.alfresco.solr.tracker;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.Node;
import static java.util.Collections.emptyMap;
/**
* Routes a document only if the shardInstance matches the provided shardId.
* The access control information is duplicated in each shard.
@@ -78,11 +80,11 @@ public class ExplicitShardIdWithDynamicPropertyRouter extends ComposableDocRoute
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return (shardProperty == null ?
Collections.emptyMap() :
Map.of(DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString()));
return shardProperty
.map(QName::getPrefixString)
.map(prefix -> Map.of(DocRouterFactory.SHARD_KEY_KEY, prefix))
.orElse(emptyMap());
}
}
@@ -18,8 +18,11 @@
*/
package org.alfresco.solr.tracker;
import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
@@ -32,7 +35,6 @@ import org.alfresco.repo.index.shard.ShardState;
import org.alfresco.solr.BoundedDeque;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.NodeReport;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.TrackerState;
import org.alfresco.solr.adapters.IOpenBitSet;
import org.alfresco.solr.client.GetNodesParameters;
@@ -88,6 +90,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
private boolean txIntervalCommitTimeServiceAvailable = false;
/** Whether the cascade tracking is enabled. */
private boolean cascadeTrackerEnabled = true;
/**
* Transaction Id range to get the first transaction in database.
* 0-2000 by default.
*/
private Pair<Long, Long> minTxnIdRange;
public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
@@ -113,6 +121,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled();
String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", "0-2000").split("-");
minTxnIdRange = new Pair<Long, Long>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1]));
// In order to apply performance optimizations, checking the availability of Repo Web Scripts is required.
// As these services are available from ACS 6.2
@@ -134,19 +144,22 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
// Try invoking txIntervalCommitTime service
try
if (shardMethod.equals(DB_ID_RANGE))
{
client.getTxIntervalCommitTime(coreName, 0l, 0l);
txIntervalCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
try
{
client.getTxIntervalCommitTime(coreName, 0l, 0l);
txIntervalCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " +
"to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
}
}
}
@@ -245,7 +258,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
state.setCheckedFirstTransactionTime(true);
log.info("No transactions found - no verification required");
firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
if (!firstTransactions.getTransactions().isEmpty())
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
@@ -257,30 +270,58 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (!state.isCheckedFirstTransactionTime())
{
firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
if (!firstTransactions.getTransactions().isEmpty())
// On Shards configured with DB_ID_RANGE, the first indexed transaction can be
// different from the first transaction in the repository as some transactions
// are skipped if they are not related with the range of the Shard.
// Getting the minCommitTime for the Shard is enough in order to check
// that the first transaction is present.
long minCommitTime = 0l;
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
long firstTxId = firstTransaction.getId();
long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
int setSize = this.infoSrv.getTxDocsSize(""+firstTxId, ""+firstTransactionCommitTime);
if (setSize == 0)
try
{
log.error("First transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter;
Pair<Long, Long> commitTimes = client.getTxIntervalCommitTime(coreName,
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
minCommitTime = commitTimes.getFirst();
}
else if (setSize == 1)
catch (NoSuchMethodException e)
{
state.setCheckedFirstTransactionTime(true);
log.info("Verified first transaction and timestamp in index");
log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage());
}
else
}
// When a Shard with DB_ID_RANGE method is empty, minCommitTime is -1.
// No firstTransaction checking is required for this case.
if (minCommitTime != -1l) {
firstTransactions = client.getTransactions(minCommitTime, 0L, null, 2000l, 1);
if (!firstTransactions.getTransactions().isEmpty())
{
log.warn("Duplicate initial transaction found with correct timestamp");
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
long firstTxId = firstTransaction.getId();
long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
int setSize = this.infoSrv.getTxDocsSize(""+firstTxId, ""+firstTransactionCommitTime);
if (setSize == 0)
{
log.error("First transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
}
else if (setSize == 1)
{
state.setCheckedFirstTransactionTime(true);
log.info("Verified first transaction and timestamp in index");
}
else
{
log.warn("Duplicate initial transaction found with correct timestamp");
}
}
}
}
@@ -290,7 +331,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
if (firstTransactions == null)
{
firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
}
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state);
@@ -361,10 +402,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Index the transaction doc after the node - if this is not found then a reindex will be done.
this.infoSrv.indexTransaction(info, false);
log.info("INDEX ACTION - Transaction {} has been indexed", transactionId);
requiresCommit = true;
trackerStats.addTxDocs(nodes.size());
}
else
{
log.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
if (docCount > batchCount)
@@ -404,6 +450,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
node.setTxnId(Long.MAX_VALUE);
this.infoSrv.indexNode(node, false);
log.info("INDEX ACTION - Node {} has been reindexed", node.getId());
requiresCommit = true;
}
checkShutdown();
@@ -454,6 +501,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Index the transaction doc after the node - if this is not found then a reindex will be done.
this.infoSrv.indexTransaction(info, true);
log.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId);
}
else
{
log.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
@@ -496,6 +548,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
node.setTxnId(Long.MAX_VALUE);
this.infoSrv.indexNode(node, true);
log.info("REINDEX ACTION - Node {} has been reindexed", node.getId());
requiresCommit = true;
}
checkShutdown();
@@ -517,6 +570,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (query != null)
{
this.infoSrv.reindexNodeByQuery(query);
log.info("REINDEX ACTION - Nodes from query {} have been reindexed", query);
requiresCommit = true;
}
checkShutdown();
@@ -541,6 +595,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByTransactionId(transactionId);
requiresCommit = true;
log.info("PURGE ACTION - Purged transactionId {}", transactionId);
}
checkShutdown();
}
@@ -561,6 +616,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByNodeId(nodeId);
log.info("PURGE ACTION - Purged nodeId {}", nodeId);
}
checkShutdown();
}
@@ -703,6 +759,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction
// for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime.
// When there isn't nodes in that range, -1 is returned as commit times
boolean shardOutOfRange = false;
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
@@ -715,23 +772,24 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Node Range it's not still available in repository
if (shardMinCommitTime == -1)
{
log.debug("#### [DB_ID_RANGE] No nodes in range [" + dbIdRangeRouter.getStartRange() + "-"
+ dbIdRangeRouter.getEndRange() + "] "
+ "exist in the repository. Skipping metadata tracking.");
return;
log.debug(
"#### [DB_ID_RANGE] No nodes in range [{}-{}] "
+ "exist in the repository. Indexing only latest transaction.",
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
if (fromCommitTime > shardMaxCommitTime)
{
log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range ["
+ dbIdRangeRouter.getStartRange() + "-" + dbIdRangeRouter.getEndRange() + "]. "
+ "Skipping metadata tracking.");
return;
log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. "
+ "Indexing only latest transaction.",
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
// Initial commit time for Node Range is greater than calculated from commit time
if (fromCommitTime < shardMinCommitTime)
{
log.debug("#### [DB_ID_RANGE] SKIPPING TRANSACTIONS FROM " + fromCommitTime + " TO "
+ shardMinCommitTime);
log.debug("#### [DB_ID_RANGE] Skipping transactions from {} to {}",
fromCommitTime, shardMinCommitTime);
fromCommitTime = shardMinCommitTime;
}
}
@@ -739,6 +797,21 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
log.debug("#### Get txn from commit time: " + fromCommitTime);
transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
state.getTimeToStopIndexing());
// When transactions are out of Shard range, only the latest transaction needs to be indexed
// in order to preserve the state up-to-date of the MetadataTracker
if (shardOutOfRange)
{
Transaction latestTransaction = new Transaction();
latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime());
latestTransaction.setId(transactions.getMaxTxnId());
transactions = new Transactions(
Arrays.asList(latestTransaction),
transactions.getMaxTxnCommitTime(),
transactions.getMaxTxnId());
log.debug("#### [DB_ID_RANGE] Latest transaction to be indexed {}", latestTransaction);
}
setLastTxCommitTimeAndTxIdInTrackerState(transactions, state);
@@ -968,9 +1041,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
setRollback(true);
setRollback(true, failCausedBy);
}
private List<Node> filterNodes(List<Node> nodes)
@@ -1093,7 +1166,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
// DB TX Count
long firstTransactionCommitTime = 0;
Transactions firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000l, 1);
if(firstTransactions.getTransactions().size() > 0)
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
@@ -26,11 +26,13 @@ import org.apache.solr.common.util.Hash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.util.Collections.emptyMap;
/**
* Routes based on a text property field.
* In this method, the value of some property is hashed and this hash is used to assign the node to a random shard.
@@ -126,12 +128,13 @@ public class PropertyRouter implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return (shardProperty == null ?
Collections.emptyMap() :
Map.of(DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString(),
DocRouterFactory.SHARD_REGEX_KEY, propertyRegEx));
return shardProperty
.map(QName::getPrefixString)
.map(prefix -> Map.of(
DocRouterFactory.SHARD_KEY_KEY, prefix,
DocRouterFactory.SHARD_REGEX_KEY, propertyRegEx))
.orElse(emptyMap());
}
}
@@ -41,9 +41,11 @@ public interface Tracker
boolean getRollback();
Throwable getRollbackCausedBy();
Properties getProps();
void setRollback(boolean rollback);
void setRollback(boolean rollback, Throwable rollbackCausedBy);
void invalidateState();
@@ -165,6 +165,7 @@ alfresco.metadata.ignore.datatype.1=app:configurations
alfresco.metadata.skipDescendantDocsForSpecificAspects=false
#alfresco.metadata.ignore.aspect.0=
# The number of matches from the index to include when rewriting wildcard search terms as an OR-ed list.
alfresco.topTermSpanRewriteLimit=1000
#
@@ -181,6 +182,13 @@ solr.suggester.minSecsBetweenBuilds=3600
#
solr.request.content.compress=false
#
# When checking repo and index consistency, first transaction is compared in both Repository and Index repositories.
# In order to get that initial transaction from database, 0-2000 range for txnId should be enough, but this parameter
# can be used when initial transaction Id is greater than 2000.
#
solr.initial.transaction.range=0-2000
#
# Limit the maximum text size of transformed content sent to the index - in bytes
@@ -165,6 +165,7 @@ alfresco.metadata.ignore.datatype.1=app:configurations
alfresco.metadata.skipDescendantDocsForSpecificAspects=false
#alfresco.metadata.ignore.aspect.0=
# The number of matches from the index to include when rewriting wildcard search terms as an OR-ed list.
alfresco.topTermSpanRewriteLimit=1000
#
@@ -181,6 +182,13 @@ solr.suggester.minSecsBetweenBuilds=3600
#
solr.request.content.compress=false
#
# When checking repo and index consistency, first transaction is compared in both Repository and Index repositories.
# In order to get that initial transaction from database, 0-2000 range for txnId should be enough, but this parameter
# can be used when initial transaction Id is greater than 2000.
#
solr.initial.transaction.range=0-2000
#
# Limit the maximum text size of transformed content sent to the index - in bytes
#
@@ -209,6 +209,15 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer
}
}
public static void explicitCommitOnAllClients() throws Exception
{
List<SolrClient> clients = getStandaloneAndShardedClients();
for (SolrClient client : clients) {
client.commit();
}
}
/**
* Gets the Default test client.
*/
@@ -61,6 +61,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -289,23 +290,6 @@ public class AlfrescoCoreAdminHandlerIT
when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT);
when(params.get(CoreAdminParams.CORE)).thenReturn(CORE_NAME);
when(params.get(ARG_TXID)).thenReturn(TX_ID);
// Set up the mock ACL tracker.
when(trackerRegistry.getTrackerForCore(CORE_NAME, AclTracker.class)).thenReturn(aclTracker);
when(aclTracker.checkIndex(Long.valueOf(TX_ID), 0L, null, null)).thenReturn(indexHealthReport);
when(indexHealthReport.getDuplicatedAclTxInIndex()).thenReturn(iOpenBitSet);
when(indexHealthReport.getAclTxInIndexButNotInDb()).thenReturn(iOpenBitSet);
when(indexHealthReport.getMissingAclTxFromIndex()).thenReturn(iOpenBitSet);
when(aclTracker.getTrackerState()).thenReturn(trackerState);
// Set up the mock metadata tracker.
when(trackerRegistry.getTrackerForCore(CORE_NAME, MetadataTracker.class)).thenReturn(metadataTracker);
when(metadataTracker.checkIndex(Long.valueOf(TX_ID), 0L, null, null)).thenReturn(metaReport);
when(metaReport.getDuplicatedTxInIndex()).thenReturn(iOpenBitSet);
when(metaReport.getTxInIndexButNotInDb()).thenReturn(iOpenBitSet);
when(metaReport.getMissingTxFromIndex()).thenReturn(iOpenBitSet);
when(metaReport.getDuplicatedLeafInIndex()).thenReturn(iOpenBitSet);
when(metaReport.getDuplicatedErrorInIndex()).thenReturn(iOpenBitSet);
when(metaReport.getDuplicatedUnindexedInIndex()).thenReturn(iOpenBitSet);
when(metadataTracker.getTrackerState()).thenReturn(trackerState);
// Call the method under test.
alfrescoCoreAdminHandler.handleCustomAction(req, rsp);
@@ -314,24 +298,27 @@ public class AlfrescoCoreAdminHandlerIT
verify(rsp).add(eq("report"), any(NamedList.class));
}
/** Check that when the transaction id is missing we get an exception. */
@Test(expected = SolrException.class)
/** Check that when the transaction id is missing we get an error message. */
@Test
public void handleCustomActionTXReportMissingTXId()
{
when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT);
alfrescoCoreAdminHandler.handleCustomAction(req, rsp);
verify(rsp, never()).add(anyString(), any());
verify(rsp).add(eq("report"), any(NamedList.class));
}
/** Check that when the core name is missing we get an exception. */
@Test(expected = SolrException.class)
/** Check that when the core name is missing we get a report for every core. */
@Test
public void handleCustomActionTXReportMissingCoreName()
{
when(params.get(CoreAdminParams.ACTION)).thenReturn(TXREPORT);
when(params.get(CoreAdminParams.CORE)).thenReturn(null);
alfrescoCoreAdminHandler.handleCustomAction(req, rsp);
// Check that a report was generated (don't look at the contents of the report though).
verify(rsp).add(eq("report"), any(NamedList.class));
}
/** Check that when an unknown action is provided we don't generate a report. */
@@ -368,7 +355,7 @@ public class AlfrescoCoreAdminHandlerIT
invalidNames.forEach(spy::setupNewDefaultCores);
verify(spy, never()).newCore(any(), anyInt(), any(), any(), anyInt(), anyInt(), anyInt(), any(), any(), any());
verify(spy, never()).newCore(any(), anyInt(), any(), any(), anyInt(), anyInt(), anyInt(), any(), any());
reset(spy);
@@ -376,16 +363,17 @@ public class AlfrescoCoreAdminHandlerIT
String commaSeparatedNames = String.join(",", invalidNames);
spy.setupNewDefaultCores(commaSeparatedNames);
verify(spy, never()).newCore(any(), anyInt(), any(), any(), anyInt(), anyInt(), anyInt(), any(), any(), any());
verify(spy, never()).newCore(any(), anyInt(), any(), any(), anyInt(), anyInt(), anyInt(), any(), any());
}
@Test
public void coreNamesAreTrimmed_oneCoreNameAtTime() {
AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() {
@Override
protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp)
protected NamedList<Object> newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties)
{
// Do nothing here otherwise we cannot spy it
return new SimpleOrderedMap<>();
}
});
@@ -399,18 +387,19 @@ public class AlfrescoCoreAdminHandlerIT
coreNames.forEach(spy::setupNewDefaultCores);
verify(spy).newCore(eq(ARCHIVE_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ARCHIVE_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null), any());
verify(spy).newCore(eq(ALFRESCO_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ALFRESCO_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null), any());
verify(spy).newCore(eq(VERSION_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(VERSION_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null), any());
verify(spy).newCore(eq(ARCHIVE_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ARCHIVE_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null));
verify(spy).newCore(eq(ALFRESCO_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ALFRESCO_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null));
verify(spy).newCore(eq(VERSION_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(VERSION_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null));
}
@Test
public void validAndInvalidCoreNames() {
AlfrescoCoreAdminHandler spy = spy(new AlfrescoCoreAdminHandler() {
@Override
protected void newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties, SolrQueryResponse rsp)
protected NamedList<Object> newCore(String coreName, int numShards, StoreRef storeRef, String templateName, int replicationFactor, int nodeInstance, int numNodes, String shardIds, Properties extraProperties)
{
// Do nothing here otherwise we cannot spy it
return new SimpleOrderedMap<>();
}
});
@@ -426,8 +415,8 @@ public class AlfrescoCoreAdminHandlerIT
String commaSeparatedNames = String.join(",", coreNames);
spy.setupNewDefaultCores(commaSeparatedNames);
verify(spy).newCore(eq(ARCHIVE_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ARCHIVE_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null), any());
verify(spy).newCore(eq(ALFRESCO_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ALFRESCO_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null), any());
verify(spy).newCore(eq(VERSION_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(VERSION_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null), any());
verify(spy).newCore(eq(ARCHIVE_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ARCHIVE_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null));
verify(spy).newCore(eq(ALFRESCO_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(ALFRESCO_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null));
verify(spy).newCore(eq(VERSION_CORE_NAME), eq(1), eq(STORE_REF_MAP.get(VERSION_CORE_NAME)), anyString(), eq(1), eq(1), eq(1), eq(null), eq(null));
}
}
@@ -122,6 +122,12 @@ public class AlfrescoSolrUtils
/**
* Get transaction.
* When getting an unique transaction for a test, don't use this constructors.
* As this produces a number that can be out of the range [1-2000], that is
* the one checked by the SOLR Core to find the initial transaction is right.
* @param deletes
* @param updates
* @return {@link Transaction}
*/
public static Transaction getTransaction(int deletes, int updates)
{
@@ -880,14 +886,16 @@ public class AlfrescoSolrUtils
TimeUnit.SECONDS.sleep(1);
if(shards > 1 )
{
NamedList vals = response.getValues();
List<String> coreNames = vals.getAll("core");
NamedList action = (NamedList) response.getValues().get("action");
List<String> coreNames = action.getAll("core");
assertEquals(shards,coreNames.size());
testingCore = getCore(coreContainer, coreNames.get(0));
}
else
{
assertEquals(coreName, response.getValues().get("core"));
NamedList action = (NamedList) response.getValues().get("action");
assertEquals(coreName, action.get("core"));
//Get a reference to the new core
testingCore = getCore(coreContainer, coreName);
}
@@ -195,7 +195,7 @@ public class AlfrescoSolrTrackerRollbackIT extends AbstractAlfrescoSolrIT
//This will prove the rollback transaction was rolled back
SOLRAPIQueueClient.transactionQueue.remove(rollbackTxn);
metadataTracker.setRollback(true);
metadataTracker.setRollback(true, new Exception("Forced rollback!"));
commitTracker.getRunLock().release();
@@ -18,29 +18,28 @@
*/
package org.alfresco.solr.tracker;
import static java.util.stream.IntStream.range;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import java.util.Properties;
import java.util.Random;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.Node;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.Properties;
import java.util.Random;
import static java.util.stream.IntStream.range;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import org.mockito.MockitoAnnotations;
/**
* {@link DBIDRangeRouter} test case.
*
* @author agazzarini
*/
@RunWith(MockitoJUnitRunner.class)
public class DBIDRangeRouterTest
{
private final Random randomizer = new Random();
@@ -55,6 +54,7 @@ public class DBIDRangeRouterTest
@Before
public void setUp()
{
MockitoAnnotations.initMocks(this);
Properties properties = new Properties();
properties.put("shard.range", "200-20000");
router = DocRouterFactory.getRouter(properties, ShardMethodEnum.DB_ID_RANGE);
@@ -0,0 +1,121 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.solr.AbstractAlfrescoDistributedIT;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclReaders;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.Transaction;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.alfresco.solr.AlfrescoSolrUtils.ancestors;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
import java.util.Properties;
/**
* Test environment having initial Transaction Id greater than default range 0-2000
* @author aborroy
*/
@SolrTestCaseJ4.SuppressSSL
@LuceneTestCase.SuppressCodecs({"Appending","Lucene3x","Lucene40","Lucene41","Lucene42","Lucene43", "Lucene44", "Lucene45","Lucene46","Lucene47","Lucene48","Lucene49"})
public class DistributedAlfrescoSolrTrackerLargeTxnIdIT extends AbstractAlfrescoDistributedIT
{
@BeforeClass
private static void initData() throws Throwable
{
Properties properties = new Properties();
// Set a wider range for Initial Transaction Id
properties.put("solr.initial.transaction.range", "0-" + Long.MAX_VALUE);
initSolrServers(2, "DistributedAlfrescoSolrTrackerLargeTxnTest", properties);
}
@AfterClass
private static void destroyData()
{
dismissSolrServers();
}
@Test
public void testTracker() throws Exception
{
putHandleDefaults();
AclChangeSet aclChangeSet = getAclChangeSet(1);
Acl acl = getAcl(aclChangeSet);
Acl acl2 = getAcl(aclChangeSet);
AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
AclReaders aclReaders2 = getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null);
// Transaction greater than 2000 is used, to check that "solr.initial.transaction.range" works as expected
Transaction txn = getTransaction(0, 2);
long txnCommitTimeMs = txn.getCommitTimeMs();
// Subtract from the commit time to go beyond hole retention
long backdatedCommitTimeMs = txnCommitTimeMs - 4600000;
txn.setCommitTimeMs(backdatedCommitTimeMs);
//Next create two nodes to update for the transaction
Node folderNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
Node fileNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
Node errorNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
// Next, create the node metadata for each node.
// Note: the error node metadata will cause an exception.
NodeMetaData folderMetaData = getNodeMetaData(folderNode, txn, acl, "mike", null, false);
NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", ancestors(folderMetaData.getNodeRef()), false);
NodeMetaData errorMetaData = getNodeMetaData(errorNode, txn, acl, "lisa", ancestors(folderMetaData.getNodeRef()), true);
// Index the transaction, nodes, and nodeMetaDatas.
// Note that the content is automatically created by the test framework.
indexTransaction(txn, asList(errorNode, folderNode, fileNode), asList(errorMetaData, folderMetaData, fileMetaData));
indexAclChangeSet(aclChangeSet, asList(acl, acl2), asList(aclReaders, aclReaders2));
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST));
BooleanQuery waitForQuery = builder.build();
waitForDocCountAllCores(waitForQuery, 1, 80000);
}
}
@@ -80,7 +80,8 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri
AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
AclReaders aclReaders2 = getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null);
Transaction txn = getTransaction(0, 2);
// Transaction between [1-2000] is required, when greater value checking the core will fail
Transaction txn = getTransaction(0, 2, 1);
long txnCommitTimeMs = txn.getCommitTimeMs();
// Subtract from the commit time to go beyond hole retention
@@ -22,8 +22,8 @@
</distributionManagement>
<properties>
<dependency.alfresco-data-model.version>8.87</dependency.alfresco-data-model.version>
<dependency.jackson.version>2.10.2</dependency.jackson.version>
<dependency.alfresco-data-model.version>8.88</dependency.alfresco-data-model.version>
<dependency.jackson.version>2.10.3</dependency.jackson.version>
</properties>
<dependencies>
@@ -67,7 +67,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.2.4</version>
<version>3.3.0</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -1254,7 +1254,7 @@ public class SOLRAPIClient
}
/**
* Returns the minimum and the maximum commit time for transactions in a node id range.
* Returns the next commit time from a given commit time.
*
* @param coreName alfresco, archive
* @param fromCommitTime initial transaction commit time
@@ -39,7 +39,7 @@ public class Transactions
private Long maxTxnId;
Transactions(List<Transaction> transactions, Long maxTxnCommitTime, Long maxTxnId)
public Transactions(List<Transaction> transactions, Long maxTxnCommitTime, Long maxTxnId)
{
this.transactions = transactions;
this.maxTxnCommitTime = maxTxnCommitTime;
+11 -4
View File
@@ -1,8 +1,15 @@
# In order to start the community edition set the value of ALFRESCO_IMAGE to alfresco/alfresco-content-repository-community
ALFRESCO_IMAGE=quay.io/alfresco/alfresco-content-repository
ALFRESCO_TAG=6.1.0-EA3
ALFRESCO_TAG=6.2.0
SHARE_IMAGE=quay.io/alfresco/alfresco-share
SHARE_TAG=6.0
SHARE_TAG=6.2.0
POSTGRES_TAG=10.1
SEARCH_TAG=latest
SEARCH_ENABLE_SPELLCHECK=false
SEARCH_TAG=test
SEARCH_ENABLE_SPELLCHECK=false
TRANSFORM_ROUTER_TAG=1.1.0
TRANSFORM_MISC_TAG=2.1.0
IMAGE_MAGICK_TAG=2.1.0
LIBREOFFICE_TAG=2.1.0
TIKA_TAG=2.1.0
PDF_RENDERER_TAG=2.1.0
NGINX_TAG=1.15.5
@@ -1,4 +1,5 @@
version: '3'
version: '3.7'
services:
alfresco:
image: ${ALFRESCO_IMAGE}:${ALFRESCO_TAG}
@@ -10,15 +11,31 @@ services:
-Ddb.url=jdbc:postgresql://postgres:5432/alfresco
-Dsolr.host=search
-Dsolr.port=8983
-Dalfresco.protocol=http
-Dsolr.secureComms=none
-Dsolr.base.url=/solr
-Dalfresco.secureComms=none
-Dsolr.baseUrl=/solr
-Dindex.subsystem.name=solr6
-Dalfresco.restApi.basicAuthScheme=true
-Dcsrf.filter.enabled=false
-Dcors.enabled=false
-Dmessaging.broker.url=\"failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true\"
-Dtransform.service.enabled=false
-Dtransform.service.url=
-Dsfs.url=
-Dlocal.transform.service.enabled=false
-DlocalTransform.libreoffice.url=
-DlocalTransform.tika.url=
-DlocalTransform.misc.url=
-DlocalTransform.pdfrenderer.url=
-DlocalTransform.imagemagick.url=
"
ports:
- "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi
- "5005:5005" #Java debugging
- "8081:8080" #Browser port for Alfresco
share:
image: ${SHARE_IMAGE}:${SHARE_TAG}
environment:
@@ -26,6 +43,7 @@ services:
- REPO_PORT=8080
ports:
- 8082:8080 #Browser port for Share
postgres:
image: postgres:${POSTGRES_TAG}
environment:
@@ -34,13 +52,12 @@ services:
- POSTGRES_DB=alfresco
ports:
- 5432:5432
search:
image: quay.io/alfresco/search-services:${SEARCH_TAG}
environment:
#Replication properties
- REPLICATION_TYPE=master
#- REPLICATION_AFTER=commit,startup- SOLR_ALFRESCO_HOST=alfresco
#- REPLICATION_CONFIG_FILES=schema.xml,stopwords.txt- SOLR_ALFRESCO_PORT=8080
# - REPLICATION_TYPE=master
#Solr needs to know how to register itself with Alfresco
- SOLR_ALFRESCO_HOST=alfresco
- SOLR_ALFRESCO_PORT=8080
@@ -55,26 +72,6 @@ services:
- ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK}
ports:
- 8083:8983 #Browser port
#search_slave:
# image: quay.io/alfresco/search-services:${SEARCH_TAG}
# environment:
# #Replication properties
# - REPLICATION_TYPE=slave
# - REPLICATION_MASTER_HOST=search
# - REPLICATION_MASTER_PORT=8983
# #- REPLICATION_MASTER_PROTOCOL=http
# #- REPLICATION_CORE_NAME=alfresco
# #- REPLICATION_POLL_INTERVAL=00:00:60
# #Solr needs to know how to register itself with Alfresco
# - SOLR_ALFRESCO_HOST=alfresco
# - SOLR_ALFRESCO_PORT=8080
# #Alfresco needs to know how to call solr
# - SOLR_SOLR_HOST=search
# - SOLR_SOLR_PORT=8983
# #Create the default alfresco and archive cores
# - SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive
# ports:
# - 8084:8983 #Browser port
activemq:
image: alfresco/alfresco-activemq:5.15.6
@@ -0,0 +1,87 @@
version: "3.7"
services:
alfresco:
image: ${ALFRESCO_IMAGE}:${ALFRESCO_TAG}
environment:
JAVA_OPTS : "
-Ddb.driver=org.postgresql.Driver
-Ddb.username=alfresco
-Ddb.password=alfresco
-Ddb.url=jdbc:postgresql://postgres:5432/alfresco
-Dsolr.host=search_slave_load_balancer
-Dsolr.port=80
-Dalfresco.protocol=http
-Dsolr.secureComms=none
-Dalfresco.secureComms=none
-Dsolr.baseUrl=/solr-slave
-Dindex.subsystem.name=solr6
-Dalfresco.restApi.basicAuthScheme=true
-Dcsrf.filter.enabled=false
-Dcors.enabled=false
-Dmessaging.broker.url=\"failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true\"
-Dtransform.service.enabled=false
-Dtransform.service.url=
-Dsfs.url=
-Dlocal.transform.service.enabled=false
-DlocalTransform.libreoffice.url=
-DlocalTransform.tika.url=
-DlocalTransform.misc.url=
-DlocalTransform.pdfrenderer.url=
-DlocalTransform.imagemagick.url=
"
# Insight Engine or Search Services
search:
image: quay.io/alfresco/search-services:${SEARCH_TAG}
environment:
#Solr needs to know how to register itself with Alfresco
- REPLICATION_TYPE=master
- SOLR_ALFRESCO_HOST=alfresco
- SOLR_ALFRESCO_PORT=8080
- REPLICATION_AFTER=commit,startup
- REPLICATION_CONFIG_FILES=schema.xml,stopwords.txt
#Alfresco needs to know how to call solr
- SOLR_SOLR_HOST=search
- SOLR_SOLR_PORT=8983
#Create the default alfresco and archive cores
- SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive
#HTTP by default
- ALFRESCO_SECURE_COMMS=none
#Enable Spellcheck by setting to true
- ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK}
ports:
- 8083:8983 #Browser port
search_slave_load_balancer:
image: nginx:${NGINX_TAG}
links:
- search_slave
volumes:
- ./master-slave/nginx.conf:/etc/nginx/conf.d/default.conf
ports:
- 8084:80
search_slave:
image: quay.io/alfresco/insight-engine:test
environment:
#Replication properties
- REPLICATION_TYPE=slave
- REPLICATION_MASTER_HOST=search
- REPLICATION_MASTER_PORT=8983
#- REPLICATION_CORE_NAME=${solr.core.name}
#- REPLICATION_MASTER_PROTOCOL=http
#- REPLICATION_POLL_INTERVAL=00:00:60
#Solr needs to know how to register itself with Alfresco
- SOLR_ALFRESCO_HOST=alfresco
- SOLR_ALFRESCO_PORT=8080
#Alfresco needs to know how to call solr
- SOLR_SOLR_HOST=search
- SOLR_SOLR_PORT=8983
#Create the default alfresco and archive cores
- SOLR_CREATE_ALFRESCO_DEFAULTS=alfresco,archive
#HTTP by default
- ALFRESCO_SECURE_COMMS=none
ports:
- 8983 #Browser port
@@ -0,0 +1,15 @@
server {
listen 80;
resolver 127.0.0.11 valid=5s;
set $upstream_endpoint http://search_slave:8983;
location / {
rewrite ^/(.*) /$1 break;
proxy_pass $upstream_endpoint;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
}
@@ -9,6 +9,7 @@ SOLR_RERANK_CONFIG_FILE=$RERANK_TEMPLATE_PATH/solrconfig.xml
SOLR_NORERANK_CONFIG_FILE=$NORERANK_TEMPLATE_PATH/solrconfig.xml
SOLR_RERANK_CORE_FILE=$RERANK_TEMPLATE_PATH/solrcore.properties
SOLR_NORERANK_CORE_FILE=$NORERANK_TEMPLATE_PATH/solrcore.properties
SOLR_CONTEXT_FILE=$PWD/solr/server/contexts/solr-jetty-context.xml
if [[ $REPLICATION_TYPE == "master" ]]; then
@@ -63,6 +64,7 @@ if [[ $REPLICATION_TYPE == "slave" ]]; then
<str name="pollInterval">'$REPLICATION_POLL_INTERVAL'<\/str>\
<\/lst>/g' $SOLR_RERANK_CONFIG_FILE $SOLR_NORERANK_CONFIG_FILE
sed -i "s/enable.alfresco.tracking=true/enable.alfresco.tracking=false\nenable.master=false\nenable.slave=true/g" $SOLR_RERANK_CORE_FILE $SOLR_NORERANK_CORE_FILE
sed -i 's/default="\/solr"/default="\/solr-slave"/g' $SOLR_CONTEXT_FILE
fi
SOLR_IN_FILE=$PWD/solr.in.sh
@@ -52,9 +52,9 @@ cxf-rt-wsdl-3.2.12.jar https://cxf.apache.org/
chemistry-opencmis-server-support-1.0.0.jar http://chemistry.apache.org/
chemistry-opencmis-server-bindings-1.0.0.jar http://chemistry.apache.org/
quartz-2.3.2.jar http://quartz-scheduler.org/
jackson-core-2.10.2.jar https://github.com/FasterXML/jackson
jackson-annotations-2.10.2.jar https://github.com/FasterXML/jackson
jackson-databind-2.10.2.jar https://github.com/FasterXML/jackson
jackson-core-2.10.3.jar https://github.com/FasterXML/jackson
jackson-annotations-2.10.3.jar https://github.com/FasterXML/jackson
jackson-databind-2.10.3.jar https://github.com/FasterXML/jackson
commons-httpclient-3.1-HTTPCLIENT-1265.jar http://jakarta.apache.org/commons/
spring-aop-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/
spring-beans-5.2.3.RELEASE.jar http://projects.spring.io/spring-framework/
@@ -98,7 +98,7 @@ javax.activation-api-1.2.0.jar https://github.com/javaee/activation/blob/master
=== BSD ===
jibx-run-1.3.1.jar http://jibx.sourceforge.net/
jibx-run-1.3.3.jar http://jibx.sourceforge.net/
dom4j-2.1.1.jar https://github.com/dom4j/dom4j/blob/master/LICENSE