inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge branch 'feature/SEARCH_2222_merge_master_in_2_0' into 'release/V2.0.x'"

Browse Source 
This reverts commit 9d498832eabc3163f5295f7d708d9bf7cae2cde3, reversing
changes made to 603da932d450f2936b6521e34cc80a1ffc4ca0a6.
This commit is contained in:
agazzarini
2020-05-07 10:38:48 +02:00
parent fc936c67ff
commit 32a1110ec7
74 changed files with 2098 additions and 1670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e-test/pom.xml
+1 -1
View File
@@ -11,7 +11,7 @@
<name>Search Analytics E2E Tests</name>
<description>Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share</description>
<properties>
<tas.rest.api.version>1.38</tas.rest.api.version>
<tas.rest.api.version>1.37</tas.rest.api.version>
<tas.cmis.api.version>1.13</tas.cmis.api.version>
<tas.utility.version>3.0.20</tas.utility.version>
<rm.version>3.3.0.1</rm.version>
e2e-test/src/main/java/org/alfresco/search/TestGroup.java
-2
View File
@@ -34,8 +34,6 @@ public class TestGroup
public static final String CONFIG_SHARDING_EXPLICIT ="CONFIG_Sharding_EXPLICIT"; // Alfresco search services using sharded environment and explicit routing
public static final String CONFIG_SHARDING_DB_ID_RANGE = "CONFIG_Sharding_DB_ID_RANGE"; // Alfresco Search Services using Sharding with DB_ID_RANGE
public static final String CONFIG_ENABLED_CASCADE_TRACKER ="Config_Enabled_Cascade_Tracker"; // Alfresco search services does not index fields related to cascaded updates
public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running
public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n
e2e-test/src/main/resources/model/music-model.xml
+1 -1
View File
@@ -66,7 +66,7 @@
<type>d:text</type>
<mandatory>false</mandatory>
<index enabled="true">
<tokenised>BOTH</tokenised>
<tokenised>TRUE</tokenised>
<facetable>false</facetable>
</index>
</property>
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java
-13
View File
@@ -1,6 +1,5 @@
package org.alfresco.test.search.functional.searchServices.cmis;
import org.alfresco.search.TestGroup;
import org.alfresco.utility.Utility;
import org.alfresco.utility.data.CustomObjectTypeProperties;
import org.alfresco.utility.data.provider.XMLDataConfig;
@@ -140,16 +139,4 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
@Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@XMLDataConfig(file = "src/test/resources/testdata/search-by-aspect-in-tree.xml")
public void executeSearchByAspectInTree(QueryModel query) throws Exception
{
String currentQuery = query.getValue()
.replace("NODE_REF[f1]", tasFolder1.getNodeRef())
.replace("NODE_REF[s1]", siteDoclibNodeRef);
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java
-16
View File
@@ -1,6 +1,5 @@
package org.alfresco.test.search.functional.searchServices.cmis;
import org.alfresco.search.TestGroup;
import org.alfresco.utility.Utility;
import org.alfresco.utility.data.CustomObjectTypeProperties;
import org.alfresco.utility.data.provider.XMLDataConfig;
@@ -101,19 +100,4 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
@Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@XMLDataConfig(file = "src/test/resources/testdata/search-by-id-in-tree.xml")
public void executeSearchByIdInTree(QueryModel query) throws Exception
{
String currentQuery = query.getValue()
.replace("NODE_REF[siteId]", siteDoclibNodeRef)
.replace("NODE_REF[d1]", tasSubFile1.getNodeRefWithoutVersion())
.replace("NODE_REF[d2]", tasSubFile2.getNodeRefWithoutVersion())
.replace("NODE_REF[f1]", tasFolder1.getNodeRef())
.replace("NODE_REF[f1-1]", tasSubFolder1.getNodeRef());
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java
+1 -2
View File
@@ -1,6 +1,5 @@
package org.alfresco.test.search.functional.searchServices.cmis;
import org.alfresco.search.TestGroup;
import org.alfresco.utility.Utility;
import org.alfresco.utility.data.provider.XMLDataConfig;
import org.alfresco.utility.data.provider.XMLTestData;
@@ -49,7 +48,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest
Utility.waitToLoopTime(getSolrWaitTimeInSeconds());
}
@Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData")
@XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml")
public void executeSearchByPathQueries(QueryModel query)
{
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java
+1 -2
View File
@@ -1,6 +1,5 @@
package org.alfresco.test.search.functional.searchServices.cmis;
import org.alfresco.search.TestGroup;
import org.alfresco.utility.Utility;
import org.alfresco.utility.data.provider.XMLDataConfig;
import org.alfresco.utility.data.provider.XMLTestDataProvider;
@@ -56,7 +55,7 @@ public class SolrSearchInTreeTests extends AbstractCmisE2ETest
dataContent.deleteSite(testSite);
}
@Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData")
@XMLDataConfig(file = "src/test/resources/testdata/search-in-tree.xml")
public void executeCMISQuery(QueryModel query) throws Exception
{
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java
+3 -5
View File
@@ -31,10 +31,8 @@ import org.alfresco.rest.search.RestRequestQueryModel;
import org.alfresco.rest.search.RestResultBucketsModel;
import org.alfresco.rest.search.SearchRequest;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.search.TestGroup;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.alfresco.utility.data.RandomData;
import org.alfresco.utility.data.provider.XMLTestDataProvider;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
@@ -95,7 +93,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest
waitForContentIndexing(htmlFile.getContent(), true);
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
public void testSearchFacetFieldsBucketExcludedWhenMinCount2() throws Exception
{
// Create Query with FacetFields: Site and Content MimeType
@@ -131,7 +129,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
public void testSearchWithFacetFieldsMinCountChecks() throws Exception
{
SearchRequest query = new SearchRequest();
@@ -206,7 +204,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest
Assert.assertNull(response.getContext().getFacetsFields());
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
public void testSearchWithFacetFieldsOnlyFacetsWhereAccess() throws Exception
{
SearchRequest query = new SearchRequest();
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java
+5 -5
View File
@@ -109,8 +109,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest
.containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "gap"));
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION,
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION,
description = "Check basic facet range search api")
@SuppressWarnings("unchecked")
public void searchWithRange()
@@ -159,8 +159,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest
assertEquals(info.get("endInclusive"), "true");
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION,
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION,
description = "Check date facet intervals search api")
@SuppressWarnings("unchecked")
public void searchWithRangeHardend()
@@ -257,7 +257,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest
query.setRanges(ranges);
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION,
description = "Check basic facet range search api")
@SuppressWarnings("unchecked")
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java
+2 -2
View File
@@ -102,8 +102,8 @@ public class FacetedSearchTest extends AbstractSearchServicesE2ETest
waitForContentIndexing(file4.getContent(), true);
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api")
@Test
@TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api")
public void searchWithQueryFaceting() throws Exception
{
SearchRequest query = new SearchRequest();
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java
+76 -132
View File
@@ -18,11 +18,9 @@
*/
package org.alfresco.test.search.functional.searchServices.search;
import static jersey.repackaged.com.google.common.collect.Sets.newHashSet;
import static org.testng.Assert.assertTrue;
import java.util.Set;
import org.alfresco.rest.search.SearchNodeModel;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.alfresco.utility.model.FileModel;
@@ -37,18 +35,12 @@ import org.testng.annotations.Test;
* Search end point Public API test with finger print.
*
* @author Michael Suzuki
* @author Meenal Bhave
*/
public class FingerPrintTest extends AbstractE2EFunctionalTest
{
private FileModel file1;
private FileModel file2;
private FolderModel folder;
private FileModel fileBanana, fileTaco, fileCat, fileDog, fileOriginal;
/**
* File to be created in dataprep with same content as fileOriginal
* File to be updated in test searchAfterVersionUpdate
* File to be reverted in test searchAfterVersionRevert
*/
private FileModel fileToBeUpdated;
@BeforeClass(alwaysRun = true)
public void dataPreparation() throws Exception
@@ -59,39 +51,30 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest
* |-- pangram-banana.txt
* |-- pangram-taco.txt
* |-- pangram-cat.txt
* |-- pangram-dog.txt
* |-- original.txt
* |-- toBeUpdated.txt
* |-- dog.txt
*/
folder = new FolderModel("The quick brown fox jumps over");
dataContent.usingUser(testUser).usingSite(testSite).createFolder(folder);
fileBanana = new FileModel("pangram-banana.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy banana");
fileTaco = new FileModel("pangram-taco.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy dog that ate the taco");
file1 = new FileModel("pangram-banana.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy banana");
file2 = new FileModel("pangram-taco.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy dog that ate the taco");
fileCat = new FileModel("pangram-cat.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy cat");
fileDog = new FileModel("pangram-dog.txt", FileType.TEXT_PLAIN, "The quick brown fox ate the lazy dog");
// fileOriginal with different content
fileOriginal = new FileModel("original.txt", FileType.TEXT_PLAIN, "This is an original file with some content");
// Index a new fileToBeUpdated with content similar to fileOriginal
fileToBeUpdated = new FileModel("toBeUpdated.txt", FileType.TEXT_PLAIN, "This is an original file with some content");
FileModel file3 = new FileModel("pangram-cat.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy cat");
FileModel file4 = new FileModel("dog.txt", FileType.TEXT_PLAIN, "The quick brown fox ate the lazy dog");
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileBanana);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileTaco);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileCat);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileDog);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileOriginal);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileToBeUpdated);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file1);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file2);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file3);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file4);
waitForContentIndexing(file4.getContent(), true);
// Additional wait implemented to remove inconsistent failures. Ref: Search-1438 for details
waitForIndexing("FINGERPRINT:" + fileDog.getNodeRefWithoutVersion(), true);
waitForIndexing("FINGERPRINT:" + fileToBeUpdated.getNodeRefWithoutVersion(), true);
waitForIndexing("FINGERPRINT:" + file2.getNodeRefWithoutVersion(), true);
waitForIndexing("FINGERPRINT:" + file3.getNodeRefWithoutVersion(), true);
}
@Test(priority = 1)
@Test
@Bug(id = "MNT-20449")
public void makeSureFingerprintQueryWorksAfterMetadataUpdate() throws Exception
{
@@ -122,125 +105,86 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest
* to the files loaded as part of this test.
* Note that for fingerprint to work it need a 5 word sequence.
*/
@Test(priority = 2)
@Test
public void search()
{
String uuid = fileBanana.getNodeRefWithoutVersion();
String uuid = file1.getNodeRefWithoutVersion();
Assert.assertNotNull(uuid);
String fingerprintQuery = String.format("FINGERPRINT:%s", uuid);
SearchResponse response = queryAsUser(testUser, fingerprintQuery);
String fingerprint = String.format("FINGERPRINT:%s", uuid);
SearchResponse response = query(fingerprint);
int count = response.getEntries().size();
assertTrue(count > 1);
Set<String> expectedNames = newHashSet();
expectedNames.add(fileBanana.getName());
expectedNames.add(fileTaco.getName());
expectedNames.add(fileCat.getName());
testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS);
for (SearchNodeModel m : response.getEntries())
{
String match = m.getModel().getName();
switch (match)
{
case "pangram.txt":
case "pangram-banana.txt":
case "pangram-taco.txt":
case "pangram-cat.txt":
break;
default:
throw new AssertionError("Not a match to an expected file: " + m.getModel().getName());
}
m.getModel().assertThat().field("name").isNot("dog.txt");
m.getModel().assertThat().field("name").isNot("cars.txt");
}
}
@Test(priority = 3)
@Test
public void searchSimilar()
{
String uuid = fileTaco.getNodeRefWithoutVersion();
String uuid = file2.getNodeRefWithoutVersion();
Assert.assertNotNull(uuid);
// In the response entity there is a score of each doc, change below threshold to bring more like or less.
String fingerprintQuery = String.format("FINGERPRINT:%s_68", uuid);
SearchResponse response = query(fingerprintQuery);
String fingerprint = String.format("FINGERPRINT:%s_68", uuid);
SearchResponse response = query(fingerprint);
int count = response.getEntries().size();
assertTrue(count > 1);
Set<String> expectedNames = newHashSet();
expectedNames.add(fileTaco.getName());
testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS);
for (SearchNodeModel m : response.getEntries())
{
switch (m.getModel().getName())
{
case "pangram.txt":
case "pangram-taco.txt":
break;
default:
throw new AssertionError("Not a match to an expected file: " + m.getModel().getName());
}
m.getModel().assertThat().field("name").isNot("pangram-banana.txt");
m.getModel().assertThat().field("name").isNot("pangram-cat.txt");
m.getModel().assertThat().field("name").isNot("dog.txt");
m.getModel().assertThat().field("name").isNot("cars.txt");
}
}
@Test(priority = 4)
@Test
public void searchSimilar67Percent()
{
String uuid = fileTaco.getNodeRefWithoutVersion();
String uuid = file2.getNodeRefWithoutVersion();
Assert.assertNotNull(uuid);
// Check that: When asked for less overlap of 67%: More documents are matched
String fingerprintQuery = String.format("FINGERPRINT:%s_67", uuid);
SearchResponse response = query(fingerprintQuery);
String fingerprint = String.format("FINGERPRINT:%s_68", uuid);
SearchResponse response = query(fingerprint);
int count = response.getEntries().size();
assertTrue(count > 1);
Set<String> expectedNames = newHashSet();
expectedNames.add(fileTaco.getName());
expectedNames.add(fileBanana.getName());
expectedNames.add(fileCat.getName());
testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS);
}
@Test(priority = 5, enabled = false)
@Bug(id = "SEARCH-2065")
public void searchAfterVersionUpdate()
{
// Check that fileToBeUpdated is found with a fingerprint query with fileOriginal: as they have similar content
boolean found = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true);
Assert.assertTrue(found, "Matching File Not found in results for Fingerprint Query with original file");
// Update content of the fileToBeUpdated to match file1 in the dataprep
String newFileContent = "The quick brown fox jumps over the updated file";
fileToBeUpdated.setContent(newFileContent);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(fileToBeUpdated).updateContent(newFileContent);
// Wair for the new version of the file to be indexed
assertTrue(waitForContentIndexing(fileToBeUpdated.getContent(), true));
// Check that fileToBeUpdated is NOT found with a fingerprint query with fileOriginal
boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false);
Assert.assertTrue(notFound, "Updated File unexpectedly found in results for Fingerprint Query with original file");
// Check that fileToBeUpdated is found with a fingerprint query with file1
found = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true);
Assert.assertTrue(found, "Update File Not found in results for Fingerprint Query with updated content");
}
@Test(priority = 6, enabled = false)
@Bug(id = "SEARCH-2065")
public void searchAfterVersionRevert() throws Exception
{
// Revert fileToBeUpdated to previous version
restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).revertVersion("1.0", "{}");
String revertedContent = restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).getVersionContent("1.2").toString();
Assert.assertEquals(revertedContent, fileOriginal.getContent(), "Reverted content does not match Original");
// Wair for the new version of the file to be indexed
assertTrue(waitForContentIndexing(fileToBeUpdated.getContent(), true));
// Check that fileToBeUpdated is found with a fingerprint query with fileOriginal
boolean found = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true);
Assert.assertTrue(found, "File not found in results for Fingerprint Query with original file after reverting version changes");
// Check that fileToBeUpdated is NOT found with a fingerprint query with file1
boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false);
Assert.assertTrue(notFound, "File appears in the results for Fingerprint Query even after reverting content changes");
}
@Test(priority = 7, enabled = false)
@Bug(id = "SEARCH-2065")
public void searchAfterVersionDelete() throws Exception
{
// Revert fileToBeUpdated to previous version
restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).deleteNodeVersion("1.2");
// Check that fileToBeUpdated is NOT found with a fingerprint query with fileOriginal
boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false);
Assert.assertTrue(notFound, "Updated File unexpectedly found in results for Fingerprint Query with original file");
// Check that fileToBeUpdated is found with a fingerprint query with file1
boolean found = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true);
Assert.assertTrue(found, "Update File Not found in results for Fingerprint Query with updated content");
for (SearchNodeModel m : response.getEntries())
{
switch (m.getModel().getName())
{
case "pangram.txt":
case "pangram-taco.txt":
case "pangram-cat.txt":
break;
default:
throw new AssertionError("Not a match to an expected file: " + m.getModel().getName());
}
m.getModel().assertThat().field("name").isNot("pangram-banana.txt");
m.getModel().assertThat().field("name").isNot("dog.txt");
m.getModel().assertThat().field("name").isNot("cars.txt");
}
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java
+2 -2
View File
@@ -119,8 +119,8 @@ public class PivotFacetedSearchTest extends AbstractSearchServicesE2ETest
assertPivotResponse(response, "creator", null);
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH},
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH },
executionType = ExecutionType.REGRESSION,
description = "Checks nested pivot using Search api")
public void searchWithNestedPivoting()
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java
+1 -1
View File
@@ -155,7 +155,7 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest
});
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test(priority = 4)
public void testSearchInFieldSITE()
{
String query = "SITE:" + testSite.getId();
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java
+3 -4
View File
@@ -29,7 +29,6 @@ import org.alfresco.rest.search.RestRequestQueryModel;
import org.alfresco.rest.search.RestResultBucketsModel;
import org.alfresco.rest.search.SearchRequest;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.search.TestGroup;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -93,7 +92,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest
*}}
*
*/
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
public void searchLevel0()
{
SearchRequest searchQuery = searchRequestWithAPATHFacet("name:*", "0");
@@ -109,7 +108,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest
* Test to test that the facet buckets are returned correctly for sub-level 1/
* Test to search for a searchString, that's unique to the test run and hence stable for any environment
*/
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
public void searchLevel0andIncludeSubLevel1()
{
SearchRequest searchQuery = searchRequestWithAPATHFacet("name:" + unique_searchString, "1/");
@@ -121,7 +120,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest
getFirstBucket(response).assertThat().field("label").contains("1/");
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
public void searchLevel2()
{
String queryString = "name:cars";
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java
+1 -2
View File
@@ -9,7 +9,6 @@ package org.alfresco.test.search.functional.searchServices.search;
import org.alfresco.rest.model.RestNodeAssociationModelCollection;
import org.alfresco.rest.model.RestNodeChildAssociationModel;
import org.alfresco.search.TestGroup;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.alfresco.utility.data.CustomObjectTypeProperties;
import org.alfresco.utility.model.FileModel;
@@ -53,7 +52,7 @@ public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest
waitForMetadataIndexing(file1.getName(), true);
}
@Test(priority = 1, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test(priority = 1)
public void testSearchPathForSecondaryAssociation() throws Exception
{
String queryPathFolder1 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() +
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java
+4 -4
View File
@@ -101,8 +101,8 @@ public class SearchTest extends AbstractSearchServicesE2ETest
response.getContext().assertThat().field("request").isNotEmpty();
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION,
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION,
description = "Tests a search request containing a sort clause.")
public void searchWithOneSortClause()
{
@@ -148,8 +148,8 @@ public class SearchTest extends AbstractSearchServicesE2ETest
* The first clause has always the same value for all matches so the test makes sure the request is correctly
* processed and the returned order is determined by the second clause.
*/
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION,
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION,
description = "Tests a search request containing a sort clause.")
public void searchWithTwoSortClauses()
{
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java
+1 -1
View File
@@ -203,7 +203,7 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest
assertTrue((Integer)metricCount.get("countValues") > count, "With the exclude filter there will be more documents than returned");
}
@Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test
@TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH},
executionType = ExecutionType.REGRESSION,
description = "Checks errors with stats with Pivot using Search api")
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java
+2 -2
View File
@@ -76,7 +76,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest
"failing while removing sharding model");
}
@Test(priority = 1, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test(priority = 1)
public void testChildPathWhenParentRenamed() throws Exception
{
// Create Parent folder
@@ -119,7 +119,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest
Assert.assertEquals(descendantCountOfOriginalName, 0, "Old path still has descendants: " + parentQuery);
}
@Test(priority = 2, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER})
@Test(priority = 2)
public void testGrandChildPathWhenGrandParentRenamed() throws Exception
{
// Create grand parent folder
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java
+6 -19
View File
@@ -475,24 +475,17 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest
checkResponseStatusOk(response);
Integer expand = response.getResponse().body().jsonPath().get("expand");
Integer expand = response.getResponse().body().jsonPath().get("expand");
// RangeCheck action only applies to DB_ID_RANGE Sharding method, so expect error in other sharding methods and success for DB_ID_RANGE
if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod()))
{
// This assertion replicates: testRangeCheckSharding, priority = 21, hence deleting that test as duplicate
// Value -1 is expected when the next shard already has nodes indexed.
Assert.assertTrue((expand == Integer.valueOf(-1) || expand == Integer.valueOf(0)), "RangeCheck should not have been allowed when not using Shard DB_ID_RANGE method,");
Integer minDbid = response.getResponse().body().jsonPath().get("minDbid");
Assert.assertTrue(minDbid > 0, "RangeCheck is not successful when not using Shard DB_ID_RANGE method,");
Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,");
}
else
{
Assert.assertEquals(expand, Integer.valueOf(-1), "RangeCheck should not have been allowed when not using Shard DB_ID_RANGE method,");
String exception = response.getResponse().body().jsonPath().get("exception");
Assert.assertEquals(exception, "ERROR: Wrong document router type:DBIDRouter", "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,");
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,");
}
}
@@ -516,18 +509,12 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest
// Expand action only applies to DB_ID_RANGE Sharding method, so expect error in other sharding methods and success for DB_ID_RANGE
if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod()))
{
// This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate
Assert.assertTrue((expand == Integer.valueOf(-1) || expand == Integer.valueOf(0)), "Expansion is not successful when not using Shard DB_ID_RANGE method,");
if (expand == Integer.valueOf(-1))
{
String exceptionExpected = "Expansion cannot occur if max DBID in the index is more then 75% of range.";
String exception = response.getResponse().body().jsonPath().get("exception");
Assert.assertEquals(exception, exceptionExpected, "Expansion failed with unexpected Exception while using DB_ID_RANGE sharding");
}
// This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate
Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,");
}
else
{
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method");
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,");
}
}
e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml
Vendored
-26
View File
@@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<testData xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xsi:noNamespaceSchemaLocation="https://gitlab.alfresco.com/tas/tas-namespaces/raw/master/input-test-data.xsd">
<queries>
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[f1]')" expectedResults="4" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]')" expectedResults="6" />
<query value="SELECT * FROM cm:dublincore where IN_TREE('NODE_REF[f1]')" expectedResults="0" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]')" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]')" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name ASC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:lastModificationDate ASC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:lastModificationDate DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:createdBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:createdBy ASC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name DESC, cmis:lastModifiedBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:lastModificationDate ASC" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:createdBy ASC" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="4" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC" expectedResults="6" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:lastModificationDate DESC" expectedResults="6" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:createdBy DESC" expectedResults="6" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="6" />
</queries>
</testData>
e2e-test/src/test/resources/testdata/search-by-aspect.xml
Vendored
+22
View File
@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<testData xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xsi:noNamespaceSchemaLocation="https://gitlab.alfresco.com/tas/tas-namespaces/raw/master/input-test-data.xsd">
<queries>
<query value="SELECT * FROM tas:tasContentAspect where tas:TextPropertyAC IS NOT NULL" expectedResults="5" />
<query value="SELECT * FROM tas:tasContentAspect where tas:TextPropertyAC IS NULL" expectedResults="1" />
@@ -57,6 +58,27 @@
<query value="SELECT * FROM tas:tasFolderAspect where IN_FOLDER('NODE_REF[f1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_FOLDER('NODE_REF[f1]') ORDER BY cmis:name DESC, cmis:lastModifiedBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_FOLDER('NODE_REF[s1]')" expectedResults="2" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[f1]')" expectedResults="4" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]')" expectedResults="6" />
<query value="SELECT * FROM cm:dublincore where IN_TREE('NODE_REF[f1]')" expectedResults="0" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]')" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]')" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name ASC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:lastModificationDate ASC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:lastModificationDate DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:createdBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:createdBy ASC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[f1]') ORDER BY cmis:name DESC, cmis:lastModifiedBy DESC" expectedResults="2" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:lastModificationDate ASC" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:createdBy ASC" expectedResults="4" />
<query value="SELECT * FROM tas:tasFolderAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="4" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC" expectedResults="6" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:lastModificationDate DESC" expectedResults="6" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:createdBy DESC" expectedResults="6" />
<query value="SELECT * FROM tas:tasContentAspect where IN_TREE('NODE_REF[s1]') ORDER BY cmis:name ASC, cmis:lastModifiedBy DESC" expectedResults="6" />
<query value="SELECT D.*, O.* FROM cmis:document as D JOIN tas:tasContentAspect AS O ON D.cmis:objectId = O.cmis:objectId" expectedResults="6" />
<query value="SELECT D.*, O.* FROM cmis:document as D JOIN tas:tasContentAspect AS O ON D.cmis:objectId = O.cmis:objectId WHERE O.tas:TextPropertyAC ='aspect content text file-2'"
expectedResults="1" />
e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml
Vendored
-7
View File
@@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<testData xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xsi:noNamespaceSchemaLocation="https://gitlab.alfresco.com/tas/tas-namespaces/raw/master/input-test-data.xsd">
<queries>
<query value="SELECT * FROM tas:document WHERE IN_TREE('NODE_REF[siteId]')" expectedResults="4" />
</queries>
</testData>
e2e-test/src/test/resources/testdata/search-by-id.xml
Vendored
+1
View File
@@ -6,6 +6,7 @@
<query value="SELECT * FROM tas:document" expectedResults="4" />
<query value="SELECT * FROM tas:folder WHERE IN_FOLDER('NODE_REF[siteId]')" expectedResults="2" />
<query value="SELECT * FROM tas:document WHERE IN_FOLDER('NODE_REF[siteId]')" expectedResults="1" />
<query value="SELECT * FROM tas:document WHERE IN_TREE('NODE_REF[siteId]')" expectedResults="4" />
<query value="SELECT * FROM cmis:document WHERE cmis:objectTypeId = 'D:tas:document'" expectedResults="4" />
<query value="SELECT * FROM cmis:document WHERE cmis:objectTypeId &lt;&gt; 'D:tas:document' AND cmis:name LIKE 'standard%'" expectedResults="2" />
<query value="SELECT * FROM cmis:document WHERE cmis:objectTypeId IN ('D:tas:document') AND cmis:name LIKE 'tas%'" expectedResults="4" />
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java
+3 -4
View File
@@ -18,8 +18,6 @@
*/
package org.alfresco.solr;
import static java.util.Collections.unmodifiableList;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -796,7 +794,8 @@ public class AlfrescoSolrDataModel implements QueryConstants
fieldName = getStoredTextField(propertyName);
}
indexedField.addField(fieldName, false, false);
FieldInstance field = new FieldInstance(fieldName, false, false);
indexedField.getFields().add(field);
}
/*
@@ -1437,7 +1436,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
public List<FieldInstance> getFields()
{
return unmodifiableList(fields);
return fields;
}
public void addField(String prefix, boolean localised, boolean sort)
search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java
+1 -1
View File
@@ -40,7 +40,7 @@ import org.apache.solr.common.params.SolrParams;
*/
public class HandlerOfResources {
protected static final List<String> DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.",
public static final List<String> DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.",
"alfresco.suggestable.property.",
"alfresco.cross.locale.property.",
"alfresco.cross.locale.datatype.");
search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java
+1 -1
View File
@@ -148,7 +148,7 @@ public interface InformationServer extends InformationServerCollectionProvider
IndexHealthReport reportIndexTransactions(Long minTxId, IOpenBitSet txIdsInDb, long maxTxId) throws IOException;
List<TenantDbId> getDocsWithUncleanContent() throws IOException;
List<TenantDbId> getDocsWithUncleanContent(int start, int rows) throws IOException;
void updateContent(TenantDbId docRef) throws Exception;
search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java
+168
View File
@@ -0,0 +1,168 @@
/*
* Copyright (C) 2005-2013 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
import java.io.Serializable;
import java.util.AbstractList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.RandomAccess;
/**
* A {@link List} implementation, backed by an array, taht supports resizing.
*
* This class supports the reuse of arrays across cache instances, reducing the number of
* temporary objects created by the Alfresco Solr indexing service.
*
* @author Alex Miller
*/
public class ResizeableArrayList<E> extends AbstractList<E> implements List<E>, RandomAccess, Cloneable, Serializable
{
private static final long serialVersionUID = 1L;
private Object[] values;
private int size;
boolean active = false;
public ResizeableArrayList()
{
this(10);
}
public ResizeableArrayList(int initialSize)
{
values = new Object[initialSize];
size = initialSize;
}
@SuppressWarnings("unchecked")
@Override
public E get(int index)
{
checkSize(index);
return (E)values[index];
}
private void checkSize(int index)
{
if (index >= size)
{
throw new ArrayIndexOutOfBoundsException("Index: " + index + ", Size: " + values.length);
}
}
/**
* Resize the underlying to at least minSize.
*
* @param minSize
* @throws IllegalStateException if the instance is not active.
*/
public void resize(int minSize)
{
isActive();
int oldSize = values.length;
if (minSize > oldSize)
{
int newSize = (oldSize * 3)/2 + 1;
if (newSize < minSize)
{
newSize = minSize;
}
// minCapacity is usually close to size, so this is a win:
values = Arrays.copyOf(values, newSize);
}
size = minSize;
}
@Override
public int size()
{
return size;
}
/**
* Copy elements from from into this instance
*/
public <T> void copyFrom(ResizeableArrayList<T> from)
{
isActive();
values = Arrays.copyOf(from.values, from.values.length);
size = from.size;
}
@SuppressWarnings("unchecked")
@Override
public E set(int index, E value)
{
checkSize(index);
isActive();
E oldValue = (E)values[index];
values[index] = value;
return oldValue;
}
@SuppressWarnings("unchecked")
@Override
protected Object clone() throws CloneNotSupportedException
{
try {
ResizeableArrayList<E> v = (ResizeableArrayList<E>) super.clone();
v.values = Arrays.copyOf(values, values.length);
v.size = size;
return v;
} catch (CloneNotSupportedException e) {
throw new InternalError();
}
}
private void isActive()
{
if (!active)
{
throw new IllegalStateException("Not active");
}
}
void activate()
{
active = true;
}
void deactivate()
{
active = false;
for (int i = 0 ; i < size ; i++)
{
values[i] = null;
}
}
/**
* Sort the elements, in-place, contained in this list using {@link Comparator#}
*
* @param comparator The {@link Comparator} to sort with
*/
@SuppressWarnings("unchecked")
public void sort(Comparator<? super E> comparator)
{
DualPivotQuickSort.sort((E[])values, 0, size -1 , comparator);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java
+31 -52
View File
@@ -20,7 +20,6 @@ package org.alfresco.solr;
import static java.util.Arrays.asList;
import static java.util.Arrays.stream;
import static java.util.Optional.empty;
import static java.util.Optional.ofNullable;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXCOMMITTIME;
@@ -74,9 +73,6 @@ import static org.alfresco.solr.AlfrescoSolrDataModel.getAclChangeSetDocumentId;
import static org.alfresco.solr.AlfrescoSolrDataModel.getAclDocumentId;
import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
import static org.alfresco.util.ISO8601DateFormat.isTimeComponentDefined;
import static org.alfresco.service.cmr.security.AuthorityType.EVERYONE;
import static org.alfresco.service.cmr.security.AuthorityType.GROUP;
import static org.alfresco.service.cmr.security.AuthorityType.GUEST;
import java.io.File;
import java.io.IOException;
@@ -825,7 +821,7 @@ public class SolrInformationServer implements InformationServer
}
@Override
public List<TenantDbId> getDocsWithUncleanContent() throws IOException
public List<TenantDbId> getDocsWithUncleanContent(int start, int rows) throws IOException
{
RefCounted<SolrIndexSearcher> refCounted = null;
try
@@ -850,7 +846,7 @@ public class SolrInformationServer implements InformationServer
* in current snapshot of the index.
*
* The code below runs every two minutes and purges transactions from the
* cleanContentCache that is more than 20 minutes old.
* cleanContentCache that is more then 20 minutes old.
*
*/
long purgeTime = System.currentTimeMillis();
@@ -884,9 +880,6 @@ public class SolrInformationServer implements InformationServer
delegatingCollector.setLastDelegate(collector);
searcher.search(documentsWithOutdatedContentQuery(), delegatingCollector);
LOGGER.debug("{}-[CORE {}] Processing {} documents with content to be indexed", Thread.currentThread().getId(), core.getName(), collector.getTotalHits());
if(collector.getTotalHits() == 0)
{
LOGGER.debug("No documents with outdated text content have been found.");
@@ -1677,11 +1670,11 @@ public class SolrInformationServer implements InformationServer
nmdp.setToNodeId(node.getId());
nmdp.setMaxResults(Integer.MAX_VALUE);
Optional<Collection<NodeMetaData>> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
Collection<NodeMetaData> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
if (nodeMetaDatas.isEmpty() || nodeMetaDatas.get().isEmpty()) return;
if (nodeMetaDatas.isEmpty()) return;
NodeMetaData nodeMetaData = nodeMetaDatas.get().iterator().next();
NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next();
if (node.getTxnId() == Long.MAX_VALUE)
{
LOGGER.debug("Node {} index request is part of a re-index.", node.getId());
@@ -1820,8 +1813,8 @@ public class SolrInformationServer implements InformationServer
nmdp.setIncludeTxnId(true);
nmdp.setMaxResults(1);
// Gets only one
Optional<Collection<NodeMetaData>> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
allNodeMetaDatas.addAll(nodeMetaDatas.orElse(Collections.emptyList()));
Collection<NodeMetaData> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
allNodeMetaDatas.addAll(nodeMetaDatas);
}
return allNodeMetaDatas;
@@ -1961,14 +1954,7 @@ public class SolrInformationServer implements InformationServer
nmdp.setIncludePaths(false);
nmdp.setIncludeParentAssociations(false);
nmdp.setMaxResults(Integer.MAX_VALUE);
Optional<Collection<NodeMetaData>> nodesMetaDataFromRepository = getNodesMetaDataFromRepository(nmdp);
if (nodesMetaDataFromRepository.isEmpty())
{
// Using exception for flow handling to jump to single node processing.
throw new Exception("Error loading node metadata from repository for bulk delete.");
}
nodeMetaDatas.addAll(nodesMetaDataFromRepository.get());
nodeMetaDatas.addAll(getNodesMetaDataFromRepository(nmdp));
}
for (NodeMetaData nodeMetaData : nodeMetaDatas)
@@ -2002,15 +1988,10 @@ public class SolrInformationServer implements InformationServer
// Fetches bulk metadata
nmdp.setMaxResults(Integer.MAX_VALUE);
Optional<Collection<NodeMetaData>> nodesMetaDataFromRepository = getNodesMetaDataFromRepository(nmdp);
if (nodesMetaDataFromRepository.isEmpty())
{
// Using exception for flow handling to jump to single node processing.
throw new Exception("Error loading node metadata from repository for bulk delete.");
}
Collection<NodeMetaData> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
NEXT_NODE:
for (NodeMetaData nodeMetaData : nodesMetaDataFromRepository.get())
for (NodeMetaData nodeMetaData : nodeMetaDatas)
{
long start = System.nanoTime();
@@ -2255,7 +2236,7 @@ public class SolrInformationServer implements InformationServer
dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields()
.stream()
.filter(field -> field.getField().startsWith("text@sd___@"))
.forEach(field -> addStringProperty(valueHolder, field, value, locale));
.forEach(field -> addStringProperty(valueHolder, field, value, locale, definition));
} else
{
dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields()
@@ -2264,7 +2245,7 @@ public class SolrInformationServer implements InformationServer
{
setUnitOfTimeFields(valueHolder, field.getField(), value.getValue(), definition.getDataType());
}
addStringProperty(valueHolder, field, value, locale);
addStringProperty(valueHolder, field, value, locale, definition);
});
}
}
@@ -3266,10 +3247,17 @@ public class SolrInformationServer implements InformationServer
*/
private String addTenantToAuthority(String authority, String tenant)
{
AuthorityType authorityType = AuthorityType.getAuthorityType(authority);
if ((authorityType == GROUP || authorityType == EVERYONE || authorityType == GUEST) && !tenant.isEmpty())
switch (AuthorityType.getAuthorityType(authority))
{
return authority + "@" + tenant;
case GROUP:
case EVERYONE:
case GUEST:
if (tenant.length() != 0)
{
return authority + "@" + tenant;
}
default:
break;
}
return authority;
}
@@ -3436,11 +3424,11 @@ public class SolrInformationServer implements InformationServer
nmdp.setIncludeTxnId(true);
nmdp.setMaxResults(1);
// Gets only one
Optional<Collection<NodeMetaData>> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
Collection<NodeMetaData> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp);
if (nodeMetaDatas.isPresent() && !nodeMetaDatas.get().isEmpty())
if (!nodeMetaDatas.isEmpty())
{
NodeMetaData nodeMetaData = nodeMetaDatas.get().iterator().next();
NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next();
// Only cascade update nods we know can not have changed and must be in this shard
// Node in the current TX will be explicitly updated in the outer loop
@@ -3749,7 +3737,7 @@ public class SolrInformationServer implements InformationServer
return "\u0000" + locale.getLanguage() + "\u0000" + property.getValue();
}
private void addStringProperty(BiConsumer<String, Object> consumer, FieldInstance field, StringPropertyValue property, PropertyValue localeProperty)
private void addStringProperty(BiConsumer<String, Object> consumer, FieldInstance field, StringPropertyValue property, PropertyValue localeProperty, PropertyDefinition definition)
{
consumer.accept(field.getField(), field.isLocalised() ? getLocalisedValue(property, localeProperty) : property.getValue());
}
@@ -3921,30 +3909,21 @@ public class SolrInformationServer implements InformationServer
return nodeMetaData;
}
/**
* Get the metadata for the specified nodes from the repository.
*
* @param parameters A parameters object containing either a list of nodes ({@link NodeMetaDataParameters#getNodeIds})
* or a node range ({@link NodeMetaDataParameters#getFromNodeId} and {@link NodeMetaDataParameters#getToNodeId}).
* @return Either the metadata returned by the repository, or null if there was a problem.
*/
private Optional<Collection<NodeMetaData>> getNodesMetaDataFromRepository(NodeMetaDataParameters parameters)
private Collection<NodeMetaData> getNodesMetaDataFromRepository(NodeMetaDataParameters parameters)
{
Collection<NodeMetaData> nodeMetaDataCollection = null;
try
{
return Optional.of(notNullOrEmpty(repositoryClient.getNodesMetaData(parameters)));
return notNullOrEmpty(repositoryClient.getNodesMetaData(parameters));
}
catch (JSONException exception)
{
// The exception has been already logged in repositoryClient and could be huge. Simply log a reference to it here.
LOGGER.debug("JSON exception swallowed by SolrInformationServer.");
return empty();
// Nothing to be done here: the exception has been already logged in repositoryClient
return Collections.emptyList();
}
catch (Exception exception)
{
LOGGER.error("Unable to get nodes metadata from repository. See the stacktrace below for further details.", exception);
return empty();
return Collections.emptyList();
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java
Vendored
+5 -5
View File
@@ -32,9 +32,9 @@ public class CacheConstants
// public static String ALFRESCO_READER_TO_ACL_IDS_CACHE = "alfrescoReaderToAclIdsCache";
// public static String ALFRESCO_DENY_TO_ACL_IDS_CACHE = "alfrescoDenyToAclIdsCache";
public final static String ALFRESCO_AUTHORITY_CACHE = "alfrescoAuthorityCache";
public final static String ALFRESCO_OWNERLOOKUP_CACHE = "alfrescoOwnerCache";
public final static String ALFRESCO_READER_CACHE = "alfrescoReaderCache";
public final static String ALFRESCO_DENIED_CACHE = "alfrescoDeniedCache";
public final static String ALFRESCO_PATH_CACHE = "alfrescoPathCache";
public static String ALFRESCO_AUTHORITY_CACHE = "alfrescoAuthorityCache";
public static String ALFRESCO_OWNERLOOKUP_CACHE = "alfrescoOwnerCache";
public static String ALFRESCO_READER_CACHE = "alfrescoReaderCache";
public static String ALFRESCO_DENIED_CACHE = "alfrescoDeniedCache";
public static String ALFRESCO_PATH_CACHE = "alfrescoPathCache";
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java
+4 -2
View File
@@ -5472,6 +5472,8 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
throw new RuntimeException("Error analyzing multiTerm term: " + part, e);
}
}
private boolean analyzeRangeTerms = true;
protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
final BytesRef start;
@@ -5480,13 +5482,13 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
if (part1 == null) {
start = null;
} else {
start = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part1) : new BytesRef(part1);
start = analyzeRangeTerms ? analyzeMultitermTerm(field, part1) : new BytesRef(part1);
}
if (part2 == null) {
end = null;
} else {
end = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part2) : new BytesRef(part2);
end = analyzeRangeTerms ? analyzeMultitermTerm(field, part2) : new BytesRef(part2);
}
final TermRangeQuery query = new TermRangeQuery(field, start, end, startInclusive, endInclusive);
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java
+4
View File
@@ -37,6 +37,10 @@ import org.apache.lucene.search.Weight;
*/
public class SolrContainerScorer extends Scorer
{
// Unused
Weight weight;
SolrContainerScorerDocIdSetIterator iterator;
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java
+59 -21
View File
@@ -43,7 +43,8 @@ public abstract class AbstractTracker implements Tracker
static final long TIME_STEP_32_DAYS_IN_MS = 1000 * 60 * 60 * 24 * 32L;
static final long TIME_STEP_1_HR_IN_MS = 60 * 60 * 1000L;
static final String SHARD_METHOD_DBID = "DB_ID";
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTracker.class);
protected final Logger logger = LoggerFactory.getLogger(getClass());
protected Properties props;
protected SOLRAPIClient client;
@@ -56,6 +57,9 @@ public abstract class AbstractTracker implements Tracker
private int maxLiveSearchers;
private volatile boolean shutdown = false;
private Semaphore runLock = new Semaphore(1, true);
private Semaphore writeLock = new Semaphore(1, true);
protected volatile TrackerState state;
protected int shardCount;
protected int shardInstance;
@@ -70,6 +74,11 @@ public abstract class AbstractTracker implements Tracker
protected final Type type;
protected final String trackerId;
/*
* A thread handler can be used by subclasses, but they have to intentionally instantiate it.
*/
ThreadHandler threadHandler;
/**
* Default constructor, strictly for testing.
*/
@@ -159,9 +168,9 @@ public abstract class AbstractTracker implements Tracker
{
String iterationId = "IT #" + System.currentTimeMillis();
if(getRunLock().availablePermits() == 0)
if(runLock.availablePermits() == 0)
{
LOGGER.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId);
logger.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId);
return;
}
@@ -171,7 +180,7 @@ public abstract class AbstractTracker implements Tracker
* The runLock ensures that for each tracker type (metadata, content, commit, cascade) only one tracker will
* be running at a time.
*/
getRunLock().acquire();
runLock.acquire();
if (state==null && Boolean.parseBoolean(System.getProperty("alfresco.test", "false")))
{
@@ -182,7 +191,7 @@ public abstract class AbstractTracker implements Tracker
{
this.state = getTrackerState();
LOGGER.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString());
logger.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString());
this.state.setRunning(true);
}
else
@@ -200,25 +209,28 @@ public abstract class AbstractTracker implements Tracker
catch(IndexTrackingShutdownException t)
{
setRollback(true, t);
LOGGER.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t);
logger.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t);
}
catch(Throwable t)
{
setRollback(true, t);
if (t instanceof SocketTimeoutException || t instanceof ConnectException)
{
LOGGER.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId);
LOGGER.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t);
logger.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId);
if (logger.isDebugEnabled())
{
logger.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t);
}
}
else
{
LOGGER.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t);
logger.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t);
}
}
}
catch (InterruptedException e)
{
LOGGER.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e);
logger.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e);
}
finally
{
@@ -228,7 +240,7 @@ public abstract class AbstractTracker implements Tracker
state.setRunning(false);
state.setCheck(false);
});
getRunLock().release();
runLock.release();
}
}
@@ -272,6 +284,30 @@ public abstract class AbstractTracker implements Tracker
}
}
/**
* Allows time for the scheduled asynchronous tasks to complete
*/
synchronized void waitForAsynchronous()
{
AbstractWorkerRunnable currentRunnable = this.threadHandler.peekHeadReindexWorker();
while (currentRunnable != null)
{
checkShutdown();
synchronized (this)
{
try
{
wait(100);
}
catch (InterruptedException e)
{
// Nothing to be done here
}
}
currentRunnable = this.threadHandler.peekHeadReindexWorker();
}
}
int getMaxLiveSearchers()
{
return maxLiveSearchers;
@@ -301,19 +337,21 @@ public abstract class AbstractTracker implements Tracker
public void shutdown()
{
setShutdown(true);
if(this.threadHandler != null)
{
threadHandler.shutDownThreadPool();
}
}
/**
* Trackers implementing this method should decide if the Write Lock is applied
* globally for every Tracker Thread (static) or locally for each running Thread
*/
public abstract Semaphore getWriteLock();
public Semaphore getWriteLock()
{
return this.writeLock;
}
/**
* Trackers implementing this method should decide if the Run Lock is applied
* globally for every Tracker Thread (static) or locally for each running Thread
*/
public abstract Semaphore getRunLock();
Semaphore getRunLock()
{
return this.runLock;
}
public Properties getProps()
{
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java → search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java
+68 -56
View File
@@ -1,56 +1,68 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
* This file is part of Alfresco
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Synchronous Tracking Worker.
*
* @author aborroy
*
*/
public abstract class AbstractWorker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractWorker.class);
public void run()
{
boolean failed = true;
Exception failCausedBy = null;
try
{
doWork();
failed = false;
}
catch (Exception e)
{
LOGGER.warn("Index tracking batch hit an unrecoverable error ", e);
failCausedBy = e;
}
finally
{
if (failed)
{
onFail(failCausedBy);
}
}
}
abstract protected void doWork() throws Exception;
abstract protected void onFail(Throwable failCausedBy);
}
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
abstract class AbstractWorkerRunnable implements Runnable
{
protected final static Logger log = LoggerFactory.getLogger(AbstractWorkerRunnable.class);
QueueHandler queueHandler;
public AbstractWorkerRunnable(QueueHandler qh)
{
this.queueHandler = qh;
}
/*
* (non-Javadoc)
* @see java.lang.Runnable#run()
*/
@Override
public void run()
{
boolean failed = true;
Exception failCausedBy = null;
try
{
doWork();
failed = false;
}
catch (Exception e)
{
log.warn("Index tracking batch hit an unrecoverable error ", e);
failCausedBy = e;
}
finally
{
// Triple check that we get the queue state right
queueHandler.removeFromQueueAndProdHead(this);
if(failed)
{
onFail(failCausedBy);
}
}
}
abstract protected void doWork() throws Exception;
abstract protected void onFail(Throwable failCausedBy);
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
+203 -166
View File
@@ -20,22 +20,13 @@ package org.alfresco.solr.tracker;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import com.google.common.collect.Lists;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.repo.index.shard.ShardMethodEnum;
@@ -65,11 +56,8 @@ public class AclTracker extends AbstractTracker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class);
private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 2000;
private static final int DEFAULT_ACL_BATCH_SIZE = 100;
private static final int DEFAULT_ACL_TRACKER_MAX_PARALLELISM = 32;
private int aclTrackerParallelism;
private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 100;
private static final int DEFAULT_ACL_BATCH_SIZE = 10;
private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE;
private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE;
@@ -81,23 +69,6 @@ public class AclTracker extends AbstractTracker
private ConcurrentLinkedQueue<Long> aclsToIndex = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclsToPurge = new ConcurrentLinkedQueue<Long>();
private DocRouter docRouter;
private ForkJoinPool forkJoinPool;
// Share run and write locks across all AclTracker threads
private static Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
/**
* Default constructor, for testing.
@@ -111,18 +82,11 @@ public class AclTracker extends AbstractTracker
String coreName, InformationServer informationServer)
{
super(p, client, coreName, informationServer, Tracker.Type.ACL);
changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize",
String.valueOf(DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE)));
aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", String.valueOf(DEFAULT_ACL_BATCH_SIZE)));
changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", "100"));
aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", "10"));
shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID);
docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod));
aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.aclTrackerMaxParallelism",
String.valueOf(DEFAULT_ACL_TRACKER_MAX_PARALLELISM)));
forkJoinPool = new ForkJoinPool(aclTrackerParallelism);
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
threadHandler = new ThreadHandler(p, coreName, "AclTracker");
}
@Override
@@ -275,7 +239,7 @@ public class AclTracker extends AbstractTracker
}
}
protected void purgeAclChangeSets() throws IOException, JSONException
protected void purgeAclChangeSets() throws AuthenticationException, IOException, JSONException
{
while (aclChangeSetsToPurge.peek() != null)
{
@@ -290,7 +254,7 @@ public class AclTracker extends AbstractTracker
}
protected void purgeAcls() throws IOException, JSONException
protected void purgeAcls() throws AuthenticationException, IOException, JSONException
{
while (aclsToPurge.peek() != null)
{
@@ -358,6 +322,10 @@ public class AclTracker extends AbstractTracker
/**
* Checks the first and last TX time
* @param state the state of this tracker
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
*/
private void checkRepoAndIndexConsistency(TrackerState state) throws AuthenticationException, IOException, JSONException
{
@@ -374,13 +342,13 @@ public class AclTracker extends AbstractTracker
AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0);
long firstChangeSetCommitTime = firstChangeSet.getCommitTimeMs();
state.setLastGoodChangeSetCommitTimeInIndex(firstChangeSetCommitTime);
setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state);
setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets, state);
}
}
if (!state.isCheckedFirstAclTransactionTime())
{
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
firstChangeSets = client.getAclChangeSets(null, 0l, null, 2000L, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
{
AclChangeSet firstAclChangeSet= firstChangeSets.getAclChangeSets().get(0);
@@ -413,10 +381,10 @@ public class AclTracker extends AbstractTracker
{
if (firstChangeSets == null)
{
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
firstChangeSets = client.getAclChangeSets(null, 0l, null, 2000L, 1);
}
setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state);
setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets, state);
Long maxChangeSetCommitTimeInRepo = firstChangeSets.getMaxChangeSetCommitTime();
Long maxChangeSetIdInRepo = firstChangeSets.getMaxChangeSetId();
if (maxChangeSetCommitTimeInRepo != null && maxChangeSetIdInRepo != null)
@@ -496,7 +464,7 @@ public class AclTracker extends AbstractTracker
}
else
{
HashSet<AclChangeSet> alreadyFound = new HashSet<>(changeSetsFound.getDeque());
HashSet<AclChangeSet> alreadyFound = new HashSet<AclChangeSet>(changeSetsFound.getDeque());
for(AclChangeSet aclChangeSet : aclChangeSets.getAclChangeSets())
{
if(!alreadyFound.contains(aclChangeSet))
@@ -531,7 +499,7 @@ public class AclTracker extends AbstractTracker
}
IOpenBitSet aclTxIdsInDb = infoSrv.getOpenBitSetInstance();
long lastAclTxCommitTime = firstChangeSetCommitTimex;
Long lastAclTxCommitTime = Long.valueOf(firstChangeSetCommitTimex);
if (fromTime != null)
{
lastAclTxCommitTime = fromTime;
@@ -541,7 +509,7 @@ public class AclTracker extends AbstractTracker
Long minAclTxId = null;
long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
AclChangeSets aclTransactions;
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<>(100);
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<AclChangeSet>(100);
DO: do
{
aclTransactions = getSomeAclChangeSets(changeSetsFound, lastAclTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
@@ -551,14 +519,14 @@ public class AclTracker extends AbstractTracker
// include
if (toTime != null)
{
if (set.getCommitTimeMs() > toTime)
if (set.getCommitTimeMs() > toTime.longValue())
{
break DO;
}
}
if (toAclTx != null)
{
if (set.getId() > toAclTx)
if (set.getId() > toAclTx.longValue())
{
break DO;
}
@@ -584,11 +552,44 @@ public class AclTracker extends AbstractTracker
return this.infoSrv.reportAclTransactionsInIndex(minAclTxId, aclTxIdsInDb, maxAclTxId);
}
public List<Node> getFullNodesForDbTransaction(Long txid)
{
try
{
GetNodesParameters gnp = new GetNodesParameters();
ArrayList<Long> txs = new ArrayList<Long>();
txs.add(txid);
gnp.setTransactionIds(txs);
gnp.setStoreProtocol(storeRef.getProtocol());
gnp.setStoreIdentifier(storeRef.getIdentifier());
return client.getNodes(gnp, Integer.MAX_VALUE);
}
catch (IOException e)
{
throw new AlfrescoRuntimeException("Failed to get nodes", e);
}
catch (JSONException e)
{
throw new AlfrescoRuntimeException("Failed to get nodes", e);
}
catch (AuthenticationException e)
{
throw new AlfrescoRuntimeException("Failed to get nodes", e);
}
}
/**
* @param acltxid Long
* @return List<Long>
**/
public List<Long> getAclsForDbAclTransaction(Long acltxid)
{
try
{
ArrayList<Long> answer = new ArrayList<>();
ArrayList<Long> answer = new ArrayList<Long>();
AclChangeSets changeSet = client.getAclChangeSets(null, acltxid, null, acltxid+1, 1);
List<Acl> acls = client.getAcls(changeSet.getAclChangeSets(), null, Integer.MAX_VALUE);
for (Acl acl : acls)
@@ -597,7 +598,15 @@ public class AclTracker extends AbstractTracker
}
return answer;
}
catch (IOException | JSONException | AuthenticationException e)
catch (IOException e)
{
throw new AlfrescoRuntimeException("Failed to get acls", e);
}
catch (JSONException e)
{
throw new AlfrescoRuntimeException("Failed to get acls", e);
}
catch (AuthenticationException e)
{
throw new AlfrescoRuntimeException("Failed to get acls", e);
}
@@ -625,14 +634,9 @@ public class AclTracker extends AbstractTracker
}
/**
* Every ACL Change Set contains a list of ACLs to be indexed.
* This method gets ACL Change Sets from Alfresco Repository to be indexed.
*
* The indexing is performed in batches of ACL Change Sets and the Tracker Status
* is updated in batched of ACLs.
*
* Tracker Status contains the Commit Time from the latest ACL Change Set indexed,
* so new operations can be retrieved from Repository starting with that time.
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
*/
protected void trackAclChangeSets() throws AuthenticationException, IOException, JSONException
{
@@ -642,10 +646,9 @@ public class AclTracker extends AbstractTracker
boolean upToDate = false;
AclChangeSets aclChangeSets;
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<AclChangeSet>(100);
HashSet<AclChangeSet> changeSetsIndexed = new LinkedHashSet<AclChangeSet>();
long totalAclCount = 0;
int aclCount;
LOGGER.info("{}-[CORE {}] <init> Tracking ACLs", Thread.currentThread().getId(), coreName);
int aclCount = 0;
do
{
@@ -661,12 +664,12 @@ public class AclTracker extends AbstractTracker
this.state = getTrackerState();
Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound,
state.getLastChangeSetCommitTimeOnServer() == 0 ? state.getLastGoodChangeSetCommitTimeInIndex()
: state.getLastChangeSetCommitTimeOnServer());
Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex());
aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
state.getTimeToStopIndexing());
setLastChangeSetIdAndCommitTimeInTrackerState(aclChangeSets, state);
if (aclChangeSets.getAclChangeSets().size() > 0)
{
LOGGER.info("{}-[CORE {}] Found {} ACL change sets after lastTxCommitTime {}, ACL Change Sets from {} to {}",
@@ -683,63 +686,90 @@ public class AclTracker extends AbstractTracker
Thread.currentThread().getId(), coreName, fromCommitTime);
}
// Ignore indexed ACL Change Sets
aclChangeSets = new AclChangeSets(aclChangeSets.getAclChangeSets().stream()
.filter(changeSet -> {
try
{
boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() &&
infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed",
Thread.currentThread().getId(), coreName, changeSet.getId());
}
return !isInIndex;
}
catch (IOException e)
{
LOGGER.warn(
"{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index",
Thread.currentThread().getId(), coreName, changeSet.getId(), e);
return true;
}
})
.collect(Collectors.toList()));
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (aclChangeSets.getAclChangeSets().stream().anyMatch(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()))
ArrayList<AclChangeSet> changeSetBatch = new ArrayList<AclChangeSet>();
for (int i = 0; i < aclChangeSets.getAclChangeSets().size(); i++)
{
break;
AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(i);
boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() &&
infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
if (isInIndex)
{
// Logging progress for large ACL Change Set tracking every 100 tracked ACLs
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Change Set Id was already indexed: {}",
Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId());
}
changeSetsFound.add(changeSet);
}
else
{
// Logging progress for ACL Change Set
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Current Change Set Id to be indexed: {}",
Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId());
}
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()) {
upToDate = true;
break;
}
changeSetBatch.add(changeSet);
if (getAclCount(changeSetBatch) > changeSetAclsBatchSize) {
aclCount += indexBatchOfChangeSets(changeSetBatch);
totalAclCount += aclCount;
for (AclChangeSet scheduled : changeSetBatch) {
changeSetsFound.add(scheduled);
changeSetsIndexed.add(scheduled);
}
changeSetBatch.clear();
}
}
if (aclCount > batchCount) {
if (super.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) {
indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state);
long endElapsed = System.nanoTime();
trackerStats.addElapsedAclTime(aclCount, endElapsed - startElapsed);
startElapsed = endElapsed;
aclCount = 0;
}
}
checkShutdown();
}
final AtomicInteger counter = new AtomicInteger();
Collection<List<AclChangeSet>> changeSetBatches = aclChangeSets.getAclChangeSets().stream()
.collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize)).values();
for (List<AclChangeSet> changeSetBatch : changeSetBatches)
{
aclCount = indexBatchOfChangeSets(changeSetBatch);
for (AclChangeSet indexed : changeSetBatch)
{
changeSetsFound.add(indexed);
if (!changeSetBatch.isEmpty()) {
if (getAclCount(changeSetBatch) > 0) {
aclCount += indexBatchOfChangeSets(changeSetBatch);
totalAclCount += aclCount;
}
// Update last committed transactions
setLastChangeSetIdAndCommitTimeInTrackerState(changeSetBatch, state);
indexAclChangeSetAfterWorker(changeSetBatch, state);
for (AclChangeSet scheduled : changeSetBatch) {
changeSetsFound.add(scheduled);
changeSetsIndexed.add(scheduled);
}
changeSetBatch.clear();
}
if(changeSetsIndexed.size() > 0)
{
indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state);
long endElapsed = System.nanoTime();
trackerStats.addElapsedAclTime(aclCount, endElapsed-startElapsed);
startElapsed = endElapsed;
aclCount = 0;
}
}
catch(InterruptedException | ExecutionException e)
catch(InterruptedException e)
{
throw new IOException(e);
}
@@ -751,39 +781,32 @@ public class AclTracker extends AbstractTracker
}
while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false));
LOGGER.info("{}-[CORE {}] <end> Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount);
LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount);
}
private void setLastChangeSetIdAndCommitTimeInTrackerState(List<AclChangeSet> aclChangeSets, TrackerState state)
private void setLastChangeSetIdAndCommitTimeInTrackerState(AclChangeSets aclChangeSets, TrackerState state)
{
if (!aclChangeSets.isEmpty())
Long maxChangeSetCommitTime = aclChangeSets.getMaxChangeSetCommitTime();
if(maxChangeSetCommitTime != null)
{
Long maxChangeSetCommitTime = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs();
if(maxChangeSetCommitTime != null)
{
state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime);
}
Long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId();
if(maxChangeSetId != null)
{
state.setLastChangeSetIdOnServer(maxChangeSetId);
}
state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime);
}
Long maxChangeSetId = aclChangeSets.getMaxChangeSetId();
if(maxChangeSetId != null)
{
state.setLastChangeSetIdOnServer(maxChangeSetId);
}
}
/**
* Index ACL Change Set transaction after ACLs has been indexed by the worker
*/
private void indexAclChangeSetAfterWorker(Collection<AclChangeSet> changeSetsIndexed, TrackerState state)
private void indexAclChangeSetAfterAsynchronous(HashSet<AclChangeSet> changeSetsIndexed, TrackerState state)
throws IOException
{
waitForAsynchronous();
for (AclChangeSet set : changeSetsIndexed)
{
infoSrv.indexAclTransaction(set, true);
super.infoSrv.indexAclTransaction(set, true);
// Acl change sets are ordered by commit time and tie-broken by id
if (set.getCommitTimeMs() > state.getLastIndexedChangeSetCommitTime()
|| set.getCommitTimeMs() == state.getLastIndexedChangeSetCommitTime()
@@ -794,6 +817,8 @@ public class AclTracker extends AbstractTracker
}
trackerStats.addChangeSetAcls(set.getAclCount());
}
changeSetsIndexed.clear();
//super.infoSrv.commit();
}
private int getAclCount(List<AclChangeSet> changeSetBatch)
@@ -806,20 +831,19 @@ public class AclTracker extends AbstractTracker
return count;
}
/**
* Index ACLs from ACL Change Sets contained in changeSetBatch
* When total ACL indexed count is greater than the specified for a single execution
* (maxAclsPerExecution), no more ACL Change Sets are processed.
*
* @param changeSetBatch List of ACL Change Sets to be indexed
* @return List of ACL Change Set indexed and Count of ACL indexed
*/
private int indexBatchOfChangeSets(List<AclChangeSet> changeSetBatch) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException {
// Exclude ACL Change Set with no ACLs inside
List<AclChangeSet> nonEmptyChangeSets = changeSetBatch.stream()
.filter(set -> set.getAclCount() > 0)
.collect(Collectors.toList());
private int indexBatchOfChangeSets(List<AclChangeSet> changeSetBatch) throws AuthenticationException, IOException, JSONException
{
int aclCount = 0;
ArrayList<AclChangeSet> nonEmptyChangeSets = new ArrayList<AclChangeSet>(changeSetBatch.size());
for (AclChangeSet set : changeSetBatch)
{
if (set.getAclCount() > 0)
{
nonEmptyChangeSets.add(set);
}
}
ArrayList<Acl> aclBatch = new ArrayList<Acl>();
List<Acl> acls = client.getAcls(nonEmptyChangeSets, null, Integer.MAX_VALUE);
if (LOGGER.isDebugEnabled())
@@ -827,27 +851,40 @@ public class AclTracker extends AbstractTracker
LOGGER.debug("{}-[CORE {}] Found {} Acls from Acl Change Sets: {}", Thread.currentThread().getId(),
coreName, acls.size(), nonEmptyChangeSets);
}
List<List<Acl>> aclBatches = Lists.partition(acls, aclBatchSize);
return forkJoinPool.submit(() ->
aclBatches.parallelStream().map(batch -> {
new AclIndexWorker(batch).run();
return batch.size();
}).reduce(0, Integer::sum)
).get();
for (Acl acl : acls)
{
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Adding ACL {} to scheduled indexing job", Thread.currentThread().getId(),
coreName, acl.toString());
}
aclBatch.add(acl);
if (aclBatch.size() > aclBatchSize)
{
aclCount += aclBatch.size();
AclIndexWorkerRunnable aiwr = new AclIndexWorkerRunnable(this.threadHandler, aclBatch);
this.threadHandler.scheduleTask(aiwr);
aclBatch = new ArrayList<Acl>();
}
}
if (aclBatch.size() > 0)
{
aclCount += aclBatch.size();
AclIndexWorkerRunnable aiwr = new AclIndexWorkerRunnable(this.threadHandler, aclBatch);
this.threadHandler.scheduleTask(aiwr);
aclBatch = new ArrayList<Acl>();
}
return aclCount;
}
/**
* ACL Indexer
*/
class AclIndexWorker extends AbstractWorker
class AclIndexWorkerRunnable extends AbstractWorkerRunnable
{
List<Acl> acls;
AclIndexWorker(List<Acl> acls)
AclIndexWorkerRunnable(QueueHandler queueHandler, List<Acl> acls)
{
super(queueHandler);
this.acls = acls;
}
@@ -870,7 +907,7 @@ public class AclTracker extends AbstractTracker
private List<Acl> filterAcls(List<Acl> acls)
{
ArrayList<Acl> filteredList = new ArrayList<>(acls.size());
ArrayList<Acl> filteredList = new ArrayList<Acl>(acls.size());
for(Acl acl : acls)
{
if(docRouter.routeAcl(shardCount, shardInstance, acl))
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java
+41 -99
View File
@@ -18,28 +18,20 @@
*/
package org.alfresco.solr.tracker;
import static java.util.stream.Collectors.joining;
import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Semaphore;
import com.google.common.collect.Lists;
import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.SOLRAPIClient;
import org.alfresco.solr.client.Transaction;
import org.apache.commons.codec.EncoderException;
import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,29 +46,8 @@ import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
public class CascadeTracker extends AbstractTracker implements Tracker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(CascadeTracker.class);
protected final static Logger log = LoggerFactory.getLogger(CascadeTracker.class);
private static final int DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM = 32;
private static final int DEFAULT_CASCADE_NODE_BATCH_SIZE = 10;
// Share run and write locks across all CascadeTracker threads
private static Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
private int cascadeBatchSize;
private ForkJoinPool forkJoinPool;
private int cascadeTrackerParallelism;
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
public CascadeTracker(Properties p, SOLRAPIClient client, String coreName,
@@ -84,15 +55,7 @@ public class CascadeTracker extends AbstractTracker implements Tracker
{
super(p, client, coreName, informationServer, Tracker.Type.CASCADE);
cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascadeTrackerMaxParallelism",
String.valueOf(DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM)));
cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascadeNodeBatchSize",
String.valueOf(DEFAULT_CASCADE_NODE_BATCH_SIZE)));;
forkJoinPool = new ForkJoinPool(cascadeTrackerParallelism);
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
threadHandler = new ThreadHandler(p, coreName, "CascadeTracker");
}
CascadeTracker()
@@ -101,7 +64,7 @@ public class CascadeTracker extends AbstractTracker implements Tracker
}
@Override
protected void doTrack(String iterationId) throws IOException, JSONException
protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException
{
// MetadataTracker must wait until ModelTracker has run
ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker();
@@ -111,36 +74,38 @@ public class CascadeTracker extends AbstractTracker implements Tracker
}
}
public void maintenance()
{
public void maintenance() throws Exception {
}
public boolean hasMaintenance() {
return false;
}
private void trackRepository(String iterationId) throws IOException, JSONException
private void trackRepository(String iterationId) throws IOException, AuthenticationException, JSONException, EncoderException
{
checkShutdown();
processCascades(iterationId);
}
private void updateTransactionsAfterWorker(List<Transaction> txsIndexed)
private void updateTransactionsAfterAsynchronous(List<Transaction> txsIndexed)
throws IOException
{
waitForAsynchronous();
for (Transaction tx : txsIndexed)
{
super.infoSrv.updateTransaction(tx);
}
}
class CascadeIndexWorker extends AbstractWorker
class CascadeIndexWorkerRunnable extends AbstractWorkerRunnable
{
InformationServer infoServer;
List<NodeMetaData> nodes;
CascadeIndexWorker(List<NodeMetaData> nodes, InformationServer infoServer)
CascadeIndexWorkerRunnable(QueueHandler queueHandler, List<NodeMetaData> nodes, InformationServer infoServer)
{
super(queueHandler);
this.infoServer = infoServer;
this.nodes = nodes;
}
@@ -154,12 +119,11 @@ public class CascadeTracker extends AbstractTracker implements Tracker
@Override
protected void onFail(Throwable failCausedBy)
{
setRollback(true, failCausedBy);
setRollback(true, failCausedBy);
}
}
public void invalidateState()
{
public void invalidateState() {
super.invalidateState();
infoSrv.setCleanCascadeTxnFloor(-1);
}
@@ -168,68 +132,52 @@ public class CascadeTracker extends AbstractTracker implements Tracker
{
int num = 50;
List<Transaction> txBatch = null;
long totalUpdatedDocs = 0;
do {
try {
getWriteLock().acquire();
txBatch = infoSrv.getCascades(num);
if (txBatch.size() > 0)
{
LOGGER.info("{}-[CORE {}] Found {} transactions, transactions from {} to {}",
Thread.currentThread().getId(),
coreName,
txBatch.size(),
txBatch.get(0),
txBatch.get(txBatch.size() - 1));
}
else
{
LOGGER.info("{}-[CORE {}] No transaction found",
Thread.currentThread().getId(), coreName);
}
if(txBatch.size() == 0) {
//No transactions to process for cascades.
return;
}
ArrayList<Long> txIds = new ArrayList<>();
Set<Long> txIdSet = new HashSet<>();
ArrayList<Long> txIds = new ArrayList<Long>();
Set<Long> txIdSet = new HashSet<Long>();
for (Transaction tx : txBatch) {
txIds.add(tx.getId());
txIdSet.add(tx.getId());
}
List<NodeMetaData> nodeMetaDatas = infoSrv.getCascadeNodes(txIds);
Integer processedCascades = 0;
if(nodeMetaDatas.size() > 0) {
List<List<NodeMetaData>> nodeBatches = Lists.partition(nodeMetaDatas, cascadeBatchSize);
LinkedList<NodeMetaData> stack = new LinkedList<NodeMetaData>();
stack.addAll(nodeMetaDatas);
int batchSize = 10;
processedCascades = forkJoinPool.submit( () ->
nodeBatches.parallelStream().map( batch -> {
CascadeIndexWorker worker = new CascadeIndexWorker(batch, infoSrv);
worker.run();
if (LOGGER.isTraceEnabled())
{
String nodes = notNullOrEmpty(batch).stream()
.map(NodeMetaData::getId)
.map(Object::toString)
.collect(joining(","));
LOGGER.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes);
}
return batch.size();
}).reduce(0, Integer::sum)
).get();
do {
List<NodeMetaData> batch = new ArrayList<NodeMetaData>();
while (batch.size() < batchSize && stack.size() > 0) {
batch.add(stack.removeFirst());
}
CascadeIndexWorkerRunnable worker = new CascadeIndexWorkerRunnable(this.threadHandler, batch, infoSrv);
if (logger.isTraceEnabled())
{
String nodes = notNullOrEmpty(batch).stream()
.map(NodeMetaData::getId)
.map(Object::toString)
.collect(joining(","));
logger.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes);
}
this.threadHandler.scheduleTask(worker);
}
while (stack.size() > 0);
}
//Update the transaction records.
updateTransactionsAfterWorker(txBatch);
totalUpdatedDocs += processedCascades;
updateTransactionsAfterAsynchronous(txBatch);
}
catch (AuthenticationException e)
{
@@ -243,17 +191,11 @@ public class CascadeTracker extends AbstractTracker implements Tracker
{
throw new IOException(e);
}
catch (ExecutionException e)
{
e.printStackTrace();
}
finally
{
getWriteLock().release();
}
} while(txBatch.size() > 0);
LOGGER.info("{}-[CORE {}] Updated {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java
+4 -24
View File
@@ -23,11 +23,8 @@ import static java.util.Optional.empty;
import static java.util.Optional.ofNullable;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import org.alfresco.solr.InformationServer;
@@ -52,21 +49,7 @@ public class CommitTracker extends AbstractTracker
private Optional<CascadeTracker> cascadeTracker = empty();
private AtomicInteger rollbackCount = new AtomicInteger(0);
protected final static Logger LOGGER = LoggerFactory.getLogger(CommitTracker.class);
// Share run and write locks across all CommitTracker threads
private static Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class);
/**
* Default constructor, for testing.
@@ -100,9 +83,6 @@ public class CommitTracker extends AbstractTracker
commitInterval = Long.parseLong(p.getProperty("alfresco.commitInterval", "60000")); // Default: commit once per minute
newSearcherInterval = Integer.parseInt(p.getProperty("alfresco.newSearcherInterval", "120000")); // Default: Open searchers every two minutes
lastSearcherOpened = lastCommit = System.currentTimeMillis();
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
}
public boolean hasMaintenance() throws Exception
@@ -214,17 +194,17 @@ public class CommitTracker extends AbstractTracker
// Log reasons why the rollback is performed
if (aclTracker.getRollbackCausedBy() != null)
{
LOGGER.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy());
log.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy());
}
if (metadataTracker.getRollbackCausedBy() != null)
{
LOGGER.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy());
log.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy());
}
}
catch (Exception e)
{
LOGGER.error("Rollback failed", e);
log.error("Rollback failed", e);
}
finally
{
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java
+45 -67
View File
@@ -16,22 +16,15 @@
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import com.google.common.collect.Lists;
import org.alfresco.solr.AlfrescoSolrDataModel.TenantDbId;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.client.SOLRAPIClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
package org.alfresco.solr.tracker;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Semaphore;
import org.alfresco.solr.AlfrescoSolrDataModel.TenantDbId;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.client.SOLRAPIClient;
import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
@@ -43,45 +36,17 @@ import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
*/
public class ContentTracker extends AbstractTracker implements Tracker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(ContentTracker.class);
private static final int DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM = 32;
private int contentTrackerParallelism;
private int contentReadBatchSize;
private int contentUpdateBatchSize;
// Share run and write locks across all ContentTracker threads
private static final Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static final Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
private ForkJoinPool forkJoinPool;
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
public ContentTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer)
public ContentTracker(Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
{
super(p, client, coreName, informationServer, Tracker.Type.CONTENT);
int DEFAULT_CONTENT_UPDATE_BATCH_SIZE = 2000;
contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize",
String.valueOf(DEFAULT_CONTENT_UPDATE_BATCH_SIZE)));
contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.contentTrackerMaxParallelism",
String.valueOf(DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM)));
forkJoinPool = new ForkJoinPool(contentTrackerParallelism);
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
contentReadBatchSize = Integer.parseInt(p.getProperty("alfresco.contentReadBatchSize", "100"));
contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", "1000"));
threadHandler = new ThreadHandler(p, coreName, "ContentTracker");
}
ContentTracker()
@@ -97,39 +62,51 @@ public class ContentTracker extends AbstractTracker implements Tracker
long startElapsed = System.nanoTime();
checkShutdown();
final int ROWS = contentReadBatchSize;
int start = 0;
long totalDocs = 0L;
checkShutdown();
while (true)
{
try
{
getWriteLock().acquire();
List<TenantDbId> docs = notNullOrEmpty(this.infoSrv.getDocsWithUncleanContent());
List<TenantDbId> docs = notNullOrEmpty(infoSrv.getDocsWithUncleanContent(start, ROWS));
if (docs.isEmpty())
{
LOGGER.trace("No unclean document has been detected in the current ContentTracker cycle.");
logger.debug("No unclean document has been detected in the current ContentTracker cycle.");
break;
}
List<List<TenantDbId>> docBatches = Lists.partition(docs, contentUpdateBatchSize);
for (List<TenantDbId> batch : docBatches)
int docsUpdatedSinceLastCommit = 0;
for (TenantDbId doc : docs)
{
Integer processedDocuments = forkJoinPool.submit(() ->
// Parallel task here, for example
batch.parallelStream().map(doc -> {
ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(doc, infoSrv);
ciwr.run();
return 1;
}).reduce(0, Integer::sum)
).get();
ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(super.threadHandler, doc, infoSrv);
super.threadHandler.scheduleTask(ciwr);
docsUpdatedSinceLastCommit++;
long endElapsed = System.nanoTime();
trackerStats.addElapsedContentTime(processedDocuments, endElapsed - startElapsed);
startElapsed = endElapsed;
if (docsUpdatedSinceLastCommit >= contentUpdateBatchSize)
{
super.waitForAsynchronous();
checkShutdown();
long endElapsed = System.nanoTime();
trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed);
startElapsed = endElapsed;
docsUpdatedSinceLastCommit = 0;
}
}
if (docsUpdatedSinceLastCommit > 0)
{
super.waitForAsynchronous();
checkShutdown();
//this.infoSrv.commit();
long endElapsed = System.nanoTime();
trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed);
}
totalDocs += docs.size();
checkShutdown();
}
@@ -139,8 +116,7 @@ public class ContentTracker extends AbstractTracker implements Tracker
}
}
LOGGER.info("{}-[CORE {}] Total number of docs with content updated: {} ", Thread.currentThread().getId(), coreName, totalDocs);
logger.info("Total number of docs with content updated: {}", totalDocs);
}
catch(Exception e)
{
@@ -164,14 +140,16 @@ public class ContentTracker extends AbstractTracker implements Tracker
this.infoSrv.setCleanContentTxnFloor(-1);
}
class ContentIndexWorkerRunnable extends AbstractWorker
class ContentIndexWorkerRunnable extends AbstractWorkerRunnable
{
InformationServer infoServer;
TenantDbId docRef;
ContentIndexWorkerRunnable(TenantDbId doc, InformationServer infoServer)
ContentIndexWorkerRunnable(QueueHandler queueHandler, TenantDbId docRef, InformationServer infoServer)
{
this.docRef = doc;
super(queueHandler);
this.docRef = docRef;
this.infoServer = infoServer;
}
@@ -187,7 +165,7 @@ public class ContentTracker extends AbstractTracker implements Tracker
protected void onFail(Throwable failCausedBy)
{
// This will be redone in future tracking operations
LOGGER.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy);
log.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy);
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java
+3 -6
View File
@@ -46,8 +46,6 @@ import org.alfresco.solr.NodeReport;
import org.alfresco.solr.TrackerState;
import org.alfresco.solr.client.SOLRAPIClient;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Optional;
@@ -65,7 +63,6 @@ import java.util.Properties;
*/
public abstract class CoreStatePublisher extends AbstractTracker
{
private static final Logger LOGGER = LoggerFactory.getLogger(CoreStatePublisher.class);
DocRouter docRouter;
private final boolean isMaster;
@@ -121,7 +118,7 @@ public abstract class CoreStatePublisher extends AbstractTracker
updateShardProperty();
if (shardProperty.isEmpty())
{
LOGGER.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName);
logger.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName);
}
});
}
@@ -137,11 +134,11 @@ public abstract class CoreStatePublisher extends AbstractTracker
{
if (updatedShardProperty.isEmpty())
{
LOGGER.warn("The model defining {} property has been disabled", shardKeyName);
logger.warn("The model defining {} property has been disabled", shardKeyName);
}
else
{
LOGGER.info("New {} property found for {}", SHARD_KEY_KEY, shardKeyName);
logger.info("New {} property found for {} ", SHARD_KEY_KEY, shardKeyName);
}
}
shardProperty = updatedShardProperty;
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java
+6 -8
View File
@@ -18,7 +18,6 @@
*/
package org.alfresco.solr.tracker;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,15 +44,14 @@ public class DocRouterFactory
LOGGER.info("Sharding via DB_ID");
return new DBIDRouter();
case DB_ID_RANGE:
if(!properties.containsKey(SHARD_RANGE_KEY))
if(properties.containsKey(SHARD_RANGE_KEY))
{
throw new AlfrescoRuntimeException("DB_ID_RANGE sharding requires the " + SHARD_RANGE_KEY + " property to be set.");
LOGGER.info("Sharding via DB_ID_RANGE");
String[] pair =properties.getProperty(SHARD_RANGE_KEY).split("-");
long start = Long.parseLong(pair[0]);
long end = Long.parseLong(pair[1]);
return new DBIDRangeRouter(start, end);
}
LOGGER.info("Sharding via DB_ID_RANGE");
String[] pair = properties.getProperty(SHARD_RANGE_KEY).split("-");
long start = Long.parseLong(pair[0]);
long end = Long.parseLong(pair[1]);
return new DBIDRangeRouter(start, end);
case ACL_ID:
LOGGER.info("Sharding via ACL_ID");
return new ACLIDMurmurRouter();
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+265 -378
View File
@@ -18,7 +18,17 @@
*/
package org.alfresco.solr.tracker;
import com.google.common.collect.Lists;
import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.repo.index.shard.ShardState;
@@ -39,41 +49,17 @@ import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Semaphore;
import java.util.stream.Collectors;
import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE;
/*
* This tracks two things: transactions and metadata nodes
* @author Ahmed Owian
*/
public class MetadataTracker extends CoreStatePublisher implements Tracker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(MetadataTracker.class);
private static final int DEFAULT_METADATA_TRACKER_MAX_PARALLELISM = 32;
private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 2000;
private static final int DEFAULT_MAX_NUMBER_OF_TRANSACTIONS = 2000;
private static final int DEFAULT_NODE_BATCH_SIZE = 50;
private static final String DEFAULT_INITIAL_TRANSACTION_RANGE = "0-2000";
private int matadataTrackerParallelism;
private int transactionDocsBatchSize;
private int nodeBatchSize;
private int maxNumberOfTransactions;
protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class);
private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100;
private static final int DEFAULT_NODE_BATCH_SIZE = 10;
private int transactionDocsBatchSize = DEFAULT_TRANSACTION_DOCS_BATCH_SIZE;
private int nodeBatchSize = DEFAULT_NODE_BATCH_SIZE;
private ConcurrentLinkedQueue<Long> transactionsToReindex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> transactionsToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> transactionsToPurge = new ConcurrentLinkedQueue<>();
@@ -84,22 +70,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
private final boolean isRunningInProduction = !Boolean.parseBoolean(System.getProperty("alfresco.test", "false"));
private ForkJoinPool forkJoinPool;
// Share run and write locks across all MetadataTracker threads
private static Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
/**
* Check if nextTxCommitTimeService is available in the repository.
* This service is used to find the next available transaction commit time from a given time,
@@ -147,22 +117,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
InformationServer informationServer, boolean checkRepoServicesAvailability)
{
super(isMaster, p, client, coreName, informationServer, Tracker.Type.METADATA);
transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize",
String.valueOf(DEFAULT_TRANSACTION_DOCS_BATCH_SIZE)));
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize",
String.valueOf(DEFAULT_NODE_BATCH_SIZE)));
maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.maxNumberOfTransactions", String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS)));
matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadataTrackerMaxParallelism",
String.valueOf(DEFAULT_METADATA_TRACKER_MAX_PARALLELISM)));
String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE).split("-");
transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100"));
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled();
minTxnIdRange = new Pair<>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1]));
forkJoinPool = new ForkJoinPool(matadataTrackerParallelism);
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", "0-2000").split("-");
minTxnIdRange = new Pair<Long, Long>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1]));
// In order to apply performance optimizations, checking the availability of Repo Web Scripts is required.
// As these services are available from ACS 6.2
@@ -176,11 +136,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
catch (NoSuchMethodException e)
{
LOGGER.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
LOGGER.error("Checking nextTxCommitTimeService failed.", e);
log.error("Checking nextTxCommitTimeService failed.", e);
}
// Try invoking txIntervalCommitTime service
@@ -193,12 +153,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
catch (NoSuchMethodException e)
{
LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " +
log.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " +
"to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage());
}
catch (Exception e)
{
LOGGER.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
}
}
}
@@ -213,6 +173,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
@Override
protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException
{
log.debug("### MetadataTracker doTrack ###");
// MetadataTracker must wait until ModelTracker has run
ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker();
if (modelTracker != null && modelTracker.hasModels())
@@ -249,11 +210,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException
{
log.debug("####### MetadataTracker trackRepository Start #######");
checkShutdown();
// Check we are tracking the correct repository
TrackerState state = super.getTrackerState();
log.debug("####### MetadataTracker check CYCLE #######");
log.debug(String.format("%s ### state: %s", coreName, state.toString()));
if(state.getTrackerCycles() == 0)
{
//We have a new tracker state so do the checks.
@@ -293,7 +256,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
state.setCheckedLastTransactionTime(true);
state.setCheckedFirstTransactionTime(true);
LOGGER.info("No transactions found - no verification required");
log.info("No transactions found - no verification required");
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
if (!firstTransactions.getTransactions().isEmpty())
@@ -301,7 +264,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
state.setLastGoodTxCommitTimeInIndex(firstTransactionCommitTime);
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions);
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state);
}
}
@@ -325,7 +288,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
catch (NoSuchMethodException e)
{
LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage());
}
}
@@ -344,20 +307,20 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (setSize == 0)
{
LOGGER.error("First transaction was not found with the correct timestamp.");
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
log.error("First transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
}
else if (setSize == 1)
{
state.setCheckedFirstTransactionTime(true);
LOGGER.info("Verified first transaction and timestamp in index");
log.info("Verified first transaction and timestamp in index");
}
else
{
LOGGER.warn("Duplicate initial transaction found with correct timestamp");
log.warn("Duplicate initial transaction found with correct timestamp");
}
}
}
@@ -371,7 +334,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
}
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions);
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state);
Long maxTxnCommitTimeInRepo = firstTransactions.getMaxTxnCommitTime();
Long maxTxnIdInRepo = firstTransactions.getMaxTxnId();
if (maxTxnCommitTimeInRepo != null && maxTxnIdInRepo != null)
@@ -379,19 +342,19 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
Transaction maxTxInIndex = this.infoSrv.getMaxTransactionIdAndCommitTimeInIndex();
if (maxTxInIndex.getCommitTimeMs() > maxTxnCommitTimeInRepo)
{
LOGGER.error("Last transaction was found in index with timestamp later than that of repository.");
LOGGER.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo);
LOGGER.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: "
log.error("Last transaction was found in index with timestamp later than that of repository.");
log.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo);
log.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: "
+ maxTxnCommitTimeInRepo);
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp");
}
else
{
state.setCheckedLastTransactionTime(true);
LOGGER.info("Verified last transaction timestamp in index less than or equal to that of repository.");
log.info("Verified last transaction timestamp in index less than or equal to that of repository.");
}
}
}
@@ -429,9 +392,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
for (Node node : nodes)
{
docCount++;
if (LOGGER.isDebugEnabled())
if (log.isDebugEnabled())
{
LOGGER.debug(node.toString());
log.debug(node.toString());
}
this.infoSrv.indexNode(node, false);
checkShutdown();
@@ -439,15 +402,14 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Index the transaction doc after the node - if this is not found then a reindex will be done.
this.infoSrv.indexTransaction(info, false);
LOGGER.info("INDEX ACTION - Transaction {} has been indexed", transactionId);
log.info("INDEX ACTION - Transaction {} has been indexed", transactionId);
requiresCommit = true;
trackerStats.addTxDocs(nodes.size());
}
else
{
LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
log.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
@@ -456,6 +418,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if(this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers())
{
checkShutdown();
//this.infoSrv.commit();
long endElapsed = System.nanoTime();
trackerStats.addElapsedNodeTime(docCount, endElapsed-startElapsed);
startElapsed = endElapsed;
@@ -487,7 +450,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
node.setTxnId(Long.MAX_VALUE);
this.infoSrv.indexNode(node, false);
LOGGER.info("INDEX ACTION - Node {} has been reindexed", node.getId());
log.info("INDEX ACTION - Node {} has been reindexed", node.getId());
requiresCommit = true;
}
checkShutdown();
@@ -528,9 +491,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
for (Node node : nodes)
{
docCount++;
if (LOGGER.isDebugEnabled())
if (log.isDebugEnabled())
{
LOGGER.debug(node.toString());
log.debug(node.toString());
}
this.infoSrv.indexNode(node, true);
checkShutdown();
@@ -538,11 +501,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Index the transaction doc after the node - if this is not found then a reindex will be done.
this.infoSrv.indexTransaction(info, true);
LOGGER.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId);
log.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId);
}
else
{
LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
log.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
@@ -585,7 +548,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
node.setTxnId(Long.MAX_VALUE);
this.infoSrv.indexNode(node, true);
LOGGER.info("REINDEX ACTION - Node {} has been reindexed", node.getId());
log.info("REINDEX ACTION - Node {} has been reindexed", node.getId());
requiresCommit = true;
}
checkShutdown();
@@ -607,7 +570,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (query != null)
{
this.infoSrv.reindexNodeByQuery(query);
LOGGER.info("REINDEX ACTION - Nodes from query {} have been reindexed", query);
log.info("REINDEX ACTION - Nodes from query {} have been reindexed", query);
requiresCommit = true;
}
checkShutdown();
@@ -632,7 +595,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByTransactionId(transactionId);
requiresCommit = true;
LOGGER.info("PURGE ACTION - Purged transactionId {}", transactionId);
log.info("PURGE ACTION - Purged transactionId {}", transactionId);
}
checkShutdown();
}
@@ -640,6 +603,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if(requiresCommit)
{
checkShutdown();
//this.infoSrv.commit();
}
}
@@ -652,30 +616,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByNodeId(nodeId);
LOGGER.info("PURGE ACTION - Purged nodeId {}", nodeId);
log.info("PURGE ACTION - Purged nodeId {}", nodeId);
}
checkShutdown();
}
}
/**
* The fromCommitTime tells getSomeTransactions() where to start, this actually fairly straight forward.
*
* What makes this code so tricky to understand is the state.getTimeToStopIndexing().
*
* There are two scenarios to keep in mind:
*
* 1) Full re-index: In this scenario the state.getTimeToStopIndexing() will never stop the indexing.
*
* 2) Up-to-date indexing: This is where state.getTimeToStopIndexing() gets interesting. In this scenario
* the Solr index is already up to date with the repo and it is tracking new transactions. The state.getTimeToStopIndexing()
* in this scenario causes the getSomeTransactions() call to stop returning results if it finds a transaction
* beyond a specific point in time. This will break out of this loop and end the tracker run.
*
* The next time the metadata tracker runs the "continueState()" method applies the "hole retention"
* to state.getLastGoodTxCommitTimeInIndex(). This causes the state.getLastGoodTxCommitTimeInIndex() to scan
* for prior transactions that might have been missed.
*/
protected Long getTxFromCommitTime(BoundedDeque<Transaction> txnsFound, long lastGoodTxCommitTimeInIndex) {
if (txnsFound.size() > 0)
{
@@ -718,6 +664,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// step forward in time until we find something or hit the time bound
// max id unbounded
Long startTime = fromCommitTime == null ? 0L : fromCommitTime;
log.debug(String.format("#### %s MetadataTracker getSomeTransactions start time: %d end: %d",
this.coreName,
startTime,
endTime));
if(startTime == 0)
{
return client.getTransactions(startTime,
@@ -727,7 +677,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
maxResults,
shardstate);
}
do
{
transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate);
@@ -739,8 +688,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
Long nextTxCommitTime = client.getNextTxCommitTime(coreName, startTime);
if (nextTxCommitTime != -1)
{
LOGGER.info("{}-[CORE {}] Advancing transactions from {} to {}",
Thread.currentThread().getId(), coreName, startTime, nextTxCommitTime);
log.info("Advancing transactions from {} to {}", startTime, nextTxCommitTime);
transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate);
}
}
@@ -752,154 +700,26 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
return transactions;
}
/**
* When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction
* for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime.
* When there isn't nodes in that range, -1 is returned as commit times
*
* @param fromCommitTime Starting commit time to get transactions from Repository
* @param txnsFound List of transactions previously found
* @return List of transactions to be indexed
*/
private Transactions getDBIDRangeTransactions(Long fromCommitTime, BoundedDeque<Transaction> txnsFound)
throws NoSuchMethodException, AuthenticationException, IOException, JSONException, EncoderException
{
boolean shardOutOfRange = false;
DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter;
Pair<Long, Long> commitTimes = client.getTxIntervalCommitTime(coreName,
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
Long shardMinCommitTime = commitTimes.getFirst();
Long shardMaxCommitTime = commitTimes.getSecond();
// Node Range it's not still available in repository
if (shardMinCommitTime == -1)
{
LOGGER.debug(
"{}-[CORE {}] [DB_ID_RANGE] No nodes in range [{}-{}] "
+ "exist in the repository. Indexing only latest transaction.",
Thread.currentThread().getId(), coreName, dbIdRangeRouter.getStartRange(),
dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
if (fromCommitTime > shardMaxCommitTime)
{
LOGGER.debug(
"{}-[CORE {}] [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. "
+ "Indexing only latest transaction.",
Thread.currentThread().getId(), coreName, dbIdRangeRouter.getStartRange(),
dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
// Initial commit time for Node Range is greater than calculated from commit time
if (fromCommitTime < shardMinCommitTime)
{
LOGGER.debug("{}-[CORE {}] [DB_ID_RANGE] Skipping transactions from {} to {}",
Thread.currentThread().getId(), coreName, fromCommitTime, shardMinCommitTime);
fromCommitTime = shardMinCommitTime;
}
Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions,
state.getTimeToStopIndexing());
// When transactions are out of Shard range, only the latest transaction needs to be indexed
// in order to preserve the state up-to-date of the MetadataTracker
if (shardOutOfRange)
{
Transaction latestTransaction = new Transaction();
latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime());
latestTransaction.setId(transactions.getMaxTxnId());
transactions = new Transactions(
Arrays.asList(latestTransaction),
transactions.getMaxTxnCommitTime(),
transactions.getMaxTxnId());
}
return transactions;
}
/**
* Remove transactions already present in SOLR index
*
* @param transactions List of transactions to be indexed
* @return List of transactions not indexed in SOLR index
*/
private Transactions removeIndexedTransactions(Transactions transactions)
{
return new Transactions(transactions.getTransactions().stream()
.filter(transaction -> {
try
{
boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() &&
infoSrv.txnInIndex(transaction.getId(), true));
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed",
Thread.currentThread().getId(), coreName, transaction.getId());
}
return !isInIndex;
}
catch (IOException e)
{
LOGGER.warn(
"{}-[CORE {}] Error catched while checking if Transaction Id {} was in index",
Thread.currentThread().getId(), coreName, transaction.getId(), e);
return true;
}
})
.collect(Collectors.toList()));
}
//fixme remove
/**
* Keep only transactions previous to node transaction Id
*
* @param transactions List of transactions from Repository
* @param node Last Node indexed in the cycle
* @return Filtered list of transactions
*/
private Transactions filterTransactionsByNode(Transactions transactions, Node node)
{
return new Transactions(transactions.getTransactions().stream()
.filter(transaction -> {
return transaction.getId() < node.getTxnId();
})
.collect(Collectors.toList()));
}
/**
* Indexing new transactions from repository in batches of "transactionDocsBatchSize" size.
*
* Additionally, the nodes inside a transaction batch are indexed in batches of "nodeBatchSize" size.
*
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
* @throws EncoderException
*/
protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException
{
long startElapsed = System.nanoTime();
boolean upToDate = false;
Transactions transactions;
BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
int totalUpdatedDocs = 0;
LOGGER.info("{}-[CORE {}] Starting metadata tracker execution", Thread.currentThread().getId(), coreName);
HashSet<Transaction> txsIndexed = new LinkedHashSet<>();
long totalUpdatedDocs = 0;
int docCount = 0;
do
{
try
{
/*
* This write lock is used to lock out the Commit Tracker. The ensures that the MetaDataTracker will
* not be indexing content while commits or rollbacks are occurring.
*/
getWriteLock().acquire();
/*
@@ -907,94 +727,184 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
* tracker state could have been invalidated due to a rollback by the CommitTracker.
* In this case the state will revert to the last transaction state record in the index.
*/
this.state = getTrackerState();
Long fromCommitTime = getTxFromCommitTime(txnsFound,
state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime());
// Get transaction list to be indexed
/*
* The fromCommitTime tells getSomeTransactions() where to start, this actually fairly straight forward.
*
* What makes this code so tricky to understand is the state.getTimeToStopIndexing().
*
* There are two scenarios to keep in mind:
*
* 1) Full re-index: In this scenario the state.getTimeToStopIndexing() will never stop the indexing.
*
* 2) Up-to-date indexing: This is where state.getTimeToStopIndexing() gets interesting. In this scenario
* the Solr index is already up to date with the repo and it is tracking new transactions. The state.getTimeToStopIndexing()
* in this scenario causes the getSomeTransactions() call to stop returning results if it finds a transaction
* beyond a specific point in time. This will break out of this loop and end the tracker run.
*
* The next time the metadata tracker runs the "continueState()" method applies the "hole retention"
* to state.getLastGoodTxCommitTimeInIndex(). This causes the state.getLastGoodTxCommitTimeInIndex() to scan
* for prior transactions that might have been missed.
*
*/
Long fromCommitTime = getTxFromCommitTime(txnsFound,
state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime());
log.debug("#### Check txnsFound : " + txnsFound.size());
log.debug("======= fromCommitTime: " + fromCommitTime);
// When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction
// for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime.
// When there isn't nodes in that range, -1 is returned as commit times
boolean shardOutOfRange = false;
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
transactions = getDBIDRangeTransactions(fromCommitTime, txnsFound);
}
else
{
transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions,
state.getTimeToStopIndexing());
DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter;
Pair<Long, Long> commitTimes = client.getTxIntervalCommitTime(coreName,
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
Long shardMinCommitTime = commitTimes.getFirst();
Long shardMaxCommitTime = commitTimes.getSecond();
// Node Range it's not still available in repository
if (shardMinCommitTime == -1)
{
log.debug(
"#### [DB_ID_RANGE] No nodes in range [{}-{}] "
+ "exist in the repository. Indexing only latest transaction.",
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
if (fromCommitTime > shardMaxCommitTime)
{
log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. "
+ "Indexing only latest transaction.",
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
// Initial commit time for Node Range is greater than calculated from commit time
if (fromCommitTime < shardMinCommitTime)
{
log.debug("#### [DB_ID_RANGE] Skipping transactions from {} to {}",
fromCommitTime, shardMinCommitTime);
fromCommitTime = shardMinCommitTime;
}
}
// Remove transactions already indexed
transactions = removeIndexedTransactions(transactions);
if (transactions.getTransactions().size() > 0)
log.debug("#### Get txn from commit time: " + fromCommitTime);
transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
state.getTimeToStopIndexing());
// When transactions are out of Shard range, only the latest transaction needs to be indexed
// in order to preserve the state up-to-date of the MetadataTracker
if (shardOutOfRange)
{
LOGGER.info("{}-[CORE {}] Found {} transactions after lastTxCommitTime {}, transactions from {} to {}",
Thread.currentThread().getId(),
coreName,
transactions.getTransactions().size(),
fromCommitTime,
transactions.getTransactions().get(0),
transactions.getTransactions().get(transactions.getTransactions().size() - 1));
}
else
{
LOGGER.info("{}-[CORE {}] No transaction found after lastTxCommitTime {}",
Thread.currentThread().getId(),
coreName,
((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime()));
Transaction latestTransaction = new Transaction();
latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime());
latestTransaction.setId(transactions.getMaxTxnId());
transactions = new Transactions(
Arrays.asList(latestTransaction),
transactions.getMaxTxnCommitTime(),
transactions.getMaxTxnId());
log.debug("#### [DB_ID_RANGE] Latest transaction to be indexed {}", latestTransaction);
}
// Group the transactions in batches of transactionDocsBatchSize (or less)
List<List<Transaction>> txBatches = new ArrayList<>();
List<Transaction> txBatch = new ArrayList<>();
setLastTxCommitTimeAndTxIdInTrackerState(transactions, state);
log.debug("Scanning transactions ...");
if (transactions.getTransactions().size() > 0) {
log.info(".... from " + transactions.getTransactions().get(0));
log.info(".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1));
} else {
log.info(".... none found after lastTxCommitTime "
+ ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state
.getLastIndexedTxCommitTime()));
}
ArrayList<Transaction> txBatch = new ArrayList<>();
for (Transaction info : transactions.getTransactions()) {
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}",
Thread.currentThread().getId(), coreName, transactions.getTransactions().size(), info.getId());
/*
* isInIndex is used to ensure transactions that are being re-pulled due to "hole retention" are not re-indexed if
* they have already been indexed.
*
* The logic in infoSrv.txnInIndex() first checks an in-memory LRUcache for the txnId. If it doesn't find it in the cache
* it checks the index. The LRUCache is only needed for txnId's that have been indexed but are not yet visible in the index for
* one of two reasons:
*
* 1) The commit tracker has not yet committed the transaction.
* 2) The txnId has been committed to the index but the new searcher has not yet been warmed.
*
* This means that to ensure txnId's are not needlessly reprocessed during hole retention, the LRUCache must be large
* enough to cover the time between when a txnId is indexed and when it becomes visible.
*/
boolean isInIndex = (infoSrv.txnInIndex(info.getId(), true) && info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime());
if (isInIndex) {
txnsFound.add(info);
} else {
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) {
upToDate = true;
break;
}
txBatch.add(info);
if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) {
docCount += indexBatchOfTransactions(txBatch);
totalUpdatedDocs += docCount;
for (Transaction scheduledTx : txBatch) {
txnsFound.add(scheduledTx);
txsIndexed.add(scheduledTx);
}
txBatch.clear();
}
}
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) {
upToDate = true;
break;
}
if (docCount > batchCount) {
indexTransactionsAfterAsynchronous(txsIndexed, state);
long endElapsed = System.nanoTime();
trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
startElapsed = endElapsed;
docCount = 0;
txBatch.add(info);
if (getUpdateAndDeleteCount(txBatch) > transactionDocsBatchSize) {
txBatches.add(txBatch);
txBatch = new ArrayList<>();
//Release the write lock allowing the commit tracker to run.
this.getWriteLock().release();
//Re-acquire the write lock and keep indexing.
this.getWriteLock().acquire();
}
}
if (!txBatch.isEmpty())
{
txBatches.add(txBatch);
checkShutdown();
}
// Index batches of transactions and the nodes updated or deleted within the transaction
for (List<Transaction> batch : txBatches)
{
// Index any remaining transactions bringing the index to a consistent state so the CommitTracker can commit if need be.
// Index nodes contained in the transactions
int docCount = indexBatchOfTransactions(batch, totalUpdatedDocs);
totalUpdatedDocs += docCount;
if (!txBatch.isEmpty()) {
if (this.getUpdateAndDeleteCount(txBatch) > 0) {
docCount += indexBatchOfTransactions(txBatch);
totalUpdatedDocs += docCount;
}
// Add the transactions as found to avoid processing them again in the next iteration
batch.forEach(transaction -> txnsFound.add(transaction));
for (Transaction scheduledTx : txBatch) {
txnsFound.add(scheduledTx);
txsIndexed.add(scheduledTx);
}
txBatch.clear();
}
// Index the transactions
indexTransactionsAfterWorker(batch);
if (txsIndexed.size() > 0) {
indexTransactionsAfterAsynchronous(txsIndexed, state);
long endElapsed = System.nanoTime();
trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
startElapsed = endElapsed;
docCount = 0;
}
setLastTxCommitTimeAndTxIdInTrackerState(transactions);
}
catch(Exception e)
{
@@ -1008,14 +918,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
while ((transactions.getTransactions().size() > 0) && (upToDate == false));
LOGGER.info("{}-[CORE {}] Tracked {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs);
log.debug("total number of docs with metadata updated: " + totalUpdatedDocs);
}
/**
* Update latest transaction indexed in MetadataTracker state
* @param transactions List of transactions indexed
*/
private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions)
private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions, TrackerState state)
{
Long maxTxnCommitTime = transactions.getMaxTxnCommitTime();
if (maxTxnCommitTime != null)
@@ -1030,17 +936,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
}
/**
* Index transactions and update state of the tracker
* @param txsIndexed List of transactions to be indexed
* @throws IOException
*/
private void indexTransactionsAfterWorker(List<Transaction> txsIndexed)
private void indexTransactionsAfterAsynchronous(HashSet<Transaction> txsIndexed, TrackerState state)
throws IOException
{
waitForAsynchronous();
for (Transaction tx : txsIndexed)
{
infoSrv.indexTransaction(tx, true);
super.infoSrv.indexTransaction(tx, true);
// Transactions are ordered by commit time and tie-broken by tx id
if (tx.getCommitTimeMs() > state.getLastIndexedTxCommitTime()
|| tx.getCommitTimeMs() == state.getLastIndexedTxCommitTime()
@@ -1052,13 +954,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
trackerStats.addTxDocs((int) (tx.getDeletes() + tx.getUpdates()));
}
txsIndexed.clear();
//super.infoSrv.commit();
}
/**
* Return the number of updated and deleted nodes in a list of transactions
* @param txs List of transactions
* @return Number of updated and deleted nodes
*/
private long getUpdateAndDeleteCount(List<Transaction> txs)
{
long count = 0;
@@ -1069,25 +967,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
return count;
}
/**
* Index a batch of transactions.
*
* Updated or deleted nodes from these transactions are also packed into batches in order to get
* the metadata of the nodes in smaller invocations to Repository
*
* @param txBatch Batch of transactions to be indexed
* @param indexedNodes Number of nodes indexed in this Tracker execution
*
* @return Number of nodes indexed and last node indexed
*
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
*/
private int indexBatchOfTransactions(List<Transaction> txBatch, int indexedNodes) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException {
// Skip transactions without modifications (updates, deletes)
ArrayList<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size());
ArrayList<Long> txIds = new ArrayList<>();
private int indexBatchOfTransactions(List<Transaction> txBatch) throws AuthenticationException, IOException, JSONException
{
int nodeCount = 0;
List<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size());
GetNodesParameters gnp = new GetNodesParameters();
List<Long> txIds = new ArrayList<>();
for (Transaction tx : txBatch)
{
if (tx.getUpdates() > 0 || tx.getDeletes() > 0)
@@ -1097,8 +982,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
}
// Get Nodes Id properties for every transaction
GetNodesParameters gnp = new GetNodesParameters();
gnp.setTransactionIds(txIds);
gnp.setStoreProtocol(storeRef.getProtocol());
gnp.setStoreIdentifier(storeRef.getIdentifier());
@@ -1107,38 +990,42 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
gnp.setCoreName(coreName);
List<Node> nodes = client.getNodes(gnp, Integer.MAX_VALUE);
if (LOGGER.isDebugEnabled())
ArrayList<Node> nodeBatch = new ArrayList<>();
for (Node node : nodes)
{
LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}", Thread.currentThread().getId(),
coreName, nodes.size(), txIds);
if (log.isDebugEnabled())
{
log.debug(node.toString());
}
nodeBatch.add(node);
if (nodeBatch.size() > nodeBatchSize)
{
nodeCount += nodeBatch.size();
NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv);
this.threadHandler.scheduleTask(niwr);
nodeBatch = new ArrayList<>();
}
}
// Group the nodes in batches of nodeBatchSize (or less)
List<List<Node>> nodeBatches = Lists.partition(nodes, nodeBatchSize);
Integer processedNodes = forkJoinPool.submit(() ->
nodeBatches.parallelStream().map(batch -> {
new NodeIndexWorker(batch, infoSrv).run();
return batch.size();
}).reduce(0, Integer::sum)).get();
return processedNodes;
if (nodeBatch.size() > 0)
{
nodeCount += nodeBatch.size();
NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv);
this.threadHandler.scheduleTask(niwr);
nodeBatch = new ArrayList<>();
}
return nodeCount;
}
/**
* Node Indexing class running synchronously all the tracking operations.
*/
class NodeIndexWorker extends AbstractWorker
class NodeIndexWorkerRunnable extends AbstractWorkerRunnable
{
InformationServer infoServer;
List<Node> nodes;
NodeIndexWorker(List<Node> nodes, InformationServer infoServer)
NodeIndexWorkerRunnable(QueueHandler queueHandler, List<Node> nodes, InformationServer infoServer)
{
super(queueHandler);
this.infoServer = infoServer;
this.nodes = nodes;
}
@@ -1300,7 +1187,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
DO: do
{
transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, endTime);
transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime);
for (Transaction info : transactions.getTransactions())
{
// include
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java
+4 -26
View File
@@ -30,8 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.alfresco.error.AlfrescoRuntimeException;
@@ -49,8 +47,6 @@ import org.alfresco.solr.client.SOLRAPIClient;
import org.alfresco.solr.config.ConfigUtil;
import org.apache.solr.core.SolrResourceLoader;
import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @startuml
@@ -86,7 +82,6 @@ import org.slf4j.LoggerFactory;
*/
public class ModelTracker extends AbstractTracker implements Tracker
{
private static final Logger LOGGER = LoggerFactory.getLogger(ModelTracker.class);
private final Set<StoreRef> indexedStores = new HashSet<>();
private final Set<StoreRef> ignoredStores = new HashSet<>();
@@ -105,36 +100,19 @@ public class ModelTracker extends AbstractTracker implements Tracker
private volatile boolean hasModels = false;
private File alfrescoModelDir;
// Share run and write locks across all ModelTracker threads
private static Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
public ModelTracker(String solrHome, Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
{
super(p, client, coreName, informationServer, Tracker.Type.MODEL);
String normalSolrHome = SolrResourceLoader.normalizeDir(solrHome);
alfrescoModelDir = new File(ConfigUtil.locateProperty("solr.model.dir", normalSolrHome+"alfrescoModels"));
LOGGER.info("Alfresco Model dir {}", alfrescoModelDir);
logger.info("Alfresco Model dir " + alfrescoModelDir);
if (!alfrescoModelDir.exists())
{
alfrescoModelDir.mkdir();
}
loadPersistedModels();
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
}
@Override
@@ -219,7 +197,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
int registeredSearcherCount = this.infoSrv.getRegisteredSearcherCount();
if (registeredSearcherCount >= getMaxLiveSearchers())
{
LOGGER.info(".... skipping tracking registered searcher count = {}", registeredSearcherCount);
logger.info(".... skipping tracking registered searcher count = " + registeredSearcherCount);
return;
}
@@ -290,7 +268,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
}
catch (Throwable t)
{
LOGGER.error("Model tracking failed for core: {}", coreName, t);
logger.error("Model tracking failed for core: "+ coreName, t);
}
}
@@ -556,7 +534,7 @@ public class ModelTracker extends AbstractTracker implements Tracker
{
loadedModels.add(modelName);
}
LOGGER.info("Loading model {}", model.getName());
logger.info("Loading model " + model.getName());
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java → search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java
+24 -30
View File
@@ -1,30 +1,24 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
/**
* Marker exception thrown for indicating a failure in obtaining / releasing a lock.
*/
public class AlfrescoLockException extends Exception
{
AlfrescoLockException(String message)
{
super(message);
}
}
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
public interface QueueHandler
{
void removeFromQueueAndProdHead(AbstractWorkerRunnable job);
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java
+1 -28
View File
@@ -26,14 +26,9 @@ import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.TrackerState;
import org.alfresco.solr.client.SOLRAPIClient;
import org.apache.commons.codec.EncoderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
/**
* Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs
@@ -52,25 +47,6 @@ import java.util.concurrent.Semaphore;
*/
public class SlaveCoreStatePublisher extends CoreStatePublisher
{
private static final Logger LOGGER = LoggerFactory.getLogger(SlaveCoreStatePublisher.class);
// Share run and write locks across all SlaveCoreStatePublisher threads
private static final Map<String, Semaphore> RUN_LOCK_BY_CORE = new ConcurrentHashMap<>();
private static final Map<String, Semaphore> WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>();
@Override
public Semaphore getWriteLock()
{
return WRITE_LOCK_BY_CORE.get(coreName);
}
@Override
public Semaphore getRunLock()
{
return RUN_LOCK_BY_CORE.get(coreName);
}
public SlaveCoreStatePublisher(
boolean isMaster,
Properties coreProperties,
@@ -79,9 +55,6 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher
SolrInformationServer informationServer)
{
super(isMaster, coreProperties, repositoryClient, name, informationServer, NODE_STATE_PUBLISHER);
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
}
@Override
@@ -94,7 +67,7 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher
}
catch (EncoderException | IOException | AuthenticationException exception )
{
LOGGER.error("Unable to publish this node state. " +
logger.error("Unable to publish this node state. " +
"A failure condition has been met during the outbound subscription message encoding process. " +
"See the stacktrace below for further details.", exception);
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java
+104
View File
@@ -0,0 +1,104 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import java.util.Properties;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.alfresco.solr.tracker.pool.DefaultTrackerPoolFactory;
import org.alfresco.solr.tracker.pool.TrackerPoolFactory;
/**
* This class handles threads for trackers.
*
* @author Ahmed Owian
*/
public class ThreadHandler implements QueueHandler
{
/** the instance that will be given out by the factory */
private ThreadPoolExecutor threadPool;
private LinkedBlockingQueue<AbstractWorkerRunnable> threadQueue = new LinkedBlockingQueue<>();
private ReentrantReadWriteLock threadLock = new ReentrantReadWriteLock(true);
public ThreadHandler(Properties p, String coreName, String trackerName)
{
// construct the instance
TrackerPoolFactory trackerPoolFactory = new DefaultTrackerPoolFactory(p, coreName, trackerName);
threadPool = trackerPoolFactory.create();
}
public void scheduleTask(AbstractWorkerRunnable awr)
{
try
{
threadLock.writeLock().lock();
// Add the runnable to the queue to ensure ordering
threadQueue.add(awr);
}
finally
{
threadLock.writeLock().unlock();
}
threadPool.execute(awr);
}
/**
* Removes the job from the queue and notifies the HEAD
*/
public void removeFromQueueAndProdHead(AbstractWorkerRunnable job)
{
try
{
threadLock.writeLock().lock();
// Remove self from head of queue
threadQueue.remove(job);
}
finally
{
threadLock.writeLock().unlock();
}
}
/**
* Read-safe method to peek at the head of the queue
*/
public AbstractWorkerRunnable peekHeadReindexWorker()
{
try
{
threadLock.readLock().lock();
return threadQueue.peek();
}
finally
{
threadLock.readLock().unlock();
}
}
public void shutDownThreadPool()
{
if (threadPool != null)
{
threadPool.shutdownNow();
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java
-5
View File
@@ -18,7 +18,6 @@
*/
package org.alfresco.solr.tracker;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
@@ -29,12 +28,8 @@ import org.slf4j.LoggerFactory;
* Generic Solr tracker job, allowing Quartz to initiate an index update from
* a {@link Tracker} regardless of specific implementation.
*
* Concurrent execution is disallowed, as no parallel work can be done when
* indexing contents from the repository.
*
* @author Matt Ward
*/
@DisallowConcurrentExecution
public class TrackerJob implements Job
{
public static final String JOBDATA_TRACKER_KEY = "TRACKER";
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java
+1 -1
View File
@@ -99,7 +99,7 @@ public class DefaultTrackerPoolFactory implements TrackerPoolFactory
{
case "AclTracker":
corePoolSize = parseConfig("alfresco.acl.tracker.corePoolSize", p, corePoolSize);
maximumPoolSize = parseConfig("alfresco.acl.tracker.maximumPoolSize", p, maximumPoolSize);
maximumPoolSize = parseConfig("alfresco.acl.tracker.maximumPoolSize", p, maximumPoolSize);
keepAliveTime = parseConfig("alfresco.acl.tracker.keepAliveTime", p, keepAliveTime);
threadPriority = parseConfig("alfresco.acl.tracker.threadPriority", p, threadPriority);
threadDaemon = parseConfigBoolean("alfresco.acl.tracker.threadDaemon", p);
search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java
+4
View File
@@ -42,7 +42,11 @@ import org.slf4j.LoggerFactory;
public class DocValueDocTransformer extends DocTransformer
{
protected final static Logger log = LoggerFactory.getLogger(DocValueDocTransformer.class);
ResultContext context;
/* (non-Javadoc)
* @see org.apache.solr.response.transform.DocTransformer#getName()
*/
search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java
+4 -4
View File
@@ -43,7 +43,7 @@ public class CoreDescriptorDecorator
private static String SECURE_COMMS_PROPERTY = "alfresco.secureComms";
public static final ImmutableList<String> SUBSTITUTABLE_PROPERTIES_SECURE =
public static ImmutableList<String> substitutablePropertiesSecure =
ImmutableList.of(
"alfresco.host",
"alfresco.port",
@@ -59,7 +59,7 @@ public class CoreDescriptorDecorator
"alfresco.encryption.ssl.keystore.provider",
"alfresco.encryption.ssl.truststore.type");
public static final ImmutableList<String> SUBSTITUTABLE_PROPERTIES_NONE =
public static ImmutableList<String> substitutablePropertiesNone =
ImmutableList.of(
"alfresco.host",
"alfresco.port",
@@ -74,11 +74,11 @@ public class CoreDescriptorDecorator
String comms = ConfigUtil.locateProperty(SECURE_COMMS_PROPERTY, "none");
if (comms.equals("https"))
{
coreProperties = SUBSTITUTABLE_PROPERTIES_SECURE;
coreProperties = substitutablePropertiesSecure;
}
else
{
coreProperties = SUBSTITUTABLE_PROPERTIES_NONE;
coreProperties = substitutablePropertiesNone;
}
try
search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties
+1 -14
View File
@@ -30,9 +30,6 @@ alfresco.hole.check.after=300000
alfresco.batch.count=5000
alfresco.recordUnindexedNodes=false
# max time (in msecs) a given tracker instance will try to acquire a lock on a given DBID
alfresco.tracker.maxNodeLockMs=120000
# encryption
# none, https
@@ -122,22 +119,12 @@ solr.maxBooleanClauses=10000
# Batch fetch
#Max number of transactions fetched by metadata tracker
#alfresco.maxNumberOfTransactions=
alfresco.transactionDocsBatchSize=2000
alfresco.transactionDocsBatchSize=500
alfresco.nodeBatchSize=100
alfresco.changeSetAclsBatchSize=500
alfresco.aclBatchSize=100
alfresco.contentReadBatchSize=100
alfresco.contentUpdateBatchSize=1000
alfresco.cascadeNodeBatchSize=10
# Trackers thread pools
#alfresco.metadataTrackerMaxParallelism=
#alfresco.aclTrackerMaxParallelism=
#alfresco.contentTrackerMaxParallelism=
#alfresco.cascadeTrackerMaxParallelism
# Warming
search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java
+1 -2
View File
@@ -731,8 +731,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer
public static void indexTransaction(Transaction transaction, List<Node> nodes, List<NodeMetaData> nodeMetaDatas)
{
// First map the nodes to a transaction.
nodes.stream().forEach(node -> node.setTxnId(transaction.getId()));
//First map the nodes to a transaction.
SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes);
//Next map a node to the NodeMetaData
search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java
+1 -1
View File
@@ -68,7 +68,7 @@ public class AlfrescoSolrReloadIT extends AbstractAlfrescoSolrIT
singletonList(acl),
singletonList(aclReaders));
int numNodes = 1000;
int numNodes = 1;
List<Node> nodes = new ArrayList<>();
List<NodeMetaData> nodeMetaDatas = new ArrayList<>();
search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java
+109
View File
@@ -0,0 +1,109 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
public class CloudTest
{
/*
public final static String QUERY = "a query";
private Cloud cloud = new Cloud();
@Mock SolrQueryRequest request;
@SuppressWarnings({ "rawtypes" })
@Test
public void testGetQueryWithZeroValues()
{
String fieldName = FIELD_DBID;
String operator = SolrInformationServer.AND;
Collection[] valueLists = new Collection[1];
valueLists[0] = new ArrayList();
String query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals("", query);
valueLists = new Collection[0];
query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals("", query);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testGetQueryWithOneValue()
{
String fieldName = FIELD_DBID;
String operator = SolrInformationServer.AND;
Collection[] valueLists = new Collection[1];
valueLists[0] = new ArrayList();
Object value = "value";
valueLists[0].add(value);
String query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals(FIELD_DBID + ":" + value, query);
}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testGetQueryWithManyValues()
{
String fieldName = FIELD_DBID;
String operator = SolrInformationServer.AND;
Collection[] valueLists = new Collection[2];
valueLists[0] = new ArrayList();
Object value1 = "value1";
valueLists[0].add(value1);
Object value2 = "value2";
valueLists[0].add(value2);
Object value3 = "value3";
valueLists[0].add(value3);
valueLists[1] = new ArrayList();
Object value4 = "value4";
valueLists[1].add(value4);
String query = cloud.getQuery(fieldName, operator, valueLists);
assertEquals(FIELD_DBID + ":" + value1 + operator + FIELD_DBID + ":" + value2 + operator
+ FIELD_DBID + ":" + value3 + operator + FIELD_DBID + ":" + value4, query);
}
@Test
public void testExists()
{
boolean exists = cloud.exists(super.selectRequestHandler, request, QUERY);
assertFalse(exists);
}
@Test
public void testGetDocList()
{
// The response is created in the getDocList method, and the aftsRequestHandler is simply mocked.
// Therefore nothing is expected to be on the response for tests, so this verifies behavior.
DocList docList = cloud.getDocList(super.aftsRequestHandler, request, QUERY);
assertNull(docList);
ArgumentCaptor<SolrQueryResponse> response = ArgumentCaptor.forClass(SolrQueryResponse.class);
verify(super.aftsRequestHandler).handleRequest(eq(request), response.capture());
assertNotNull(response.getValue());
}
@Test
public void testSelect()
{
SolrParams params = new ModifiableSolrParams(request.getParams());
ResultContext rc = cloud.getResultContext(super.aftsRequestHandler, request, params);
assertNull(rc);
}
*/
}
search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java
-1
View File
@@ -3,7 +3,6 @@ package org.alfresco.solr;
import static org.alfresco.solr.AlfrescoSolrUtils.createCoreUsingTemplate;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.alfresco.solr.basics.RandomSupplier;
import org.alfresco.solr.client.SOLRAPIQueueClient;
import org.apache.commons.io.FileUtils;
search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java
+22 -41
View File
@@ -1,6 +1,5 @@
package org.alfresco.solr.basics;
import com.google.common.collect.Sets;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
@@ -13,10 +12,8 @@ import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class SolrResponsesComparator
{
@@ -42,7 +39,7 @@ public class SolrResponsesComparator
* Puts default values for handle
*/
public void putHandleDefaults() {
handle.put("[explain]", SKIPVAL);
handle.put("explain", SKIPVAL);
handle.put("timestamp", SKIPVAL);
handle.put("score", SKIPVAL);
handle.put("wt", SKIP);
@@ -54,7 +51,6 @@ public class SolrResponsesComparator
handle.put("_version_", SKIP);
handle.put("_original_parameters_", SKIP);
handle.put("spellcheck-extras", SKIP); // No longer used can be removed in Solr 6.
handle.put("FIELDS", UNORDERED);
}
@@ -89,6 +85,27 @@ public class SolrResponsesComparator
public void compareResponses(QueryResponse a, QueryResponse b)
{
if (System.getProperty("remove.version.field") != null)
{
// we don't care if one has a version and the other doesnt -
// control vs distrib
// TODO: this should prob be done by adding an ignore on _version_
// rather than mutating the responses?
if (a.getResults() != null)
{
for (SolrDocument doc : a.getResults())
{
doc.removeFields("_version_");
}
}
if (b.getResults() != null)
{
for (SolrDocument doc : b.getResults())
{
doc.removeFields("_version_");
}
}
}
compareSolrResponses(a, b);
}
@@ -216,29 +233,6 @@ public class SolrResponsesComparator
return compare1(b, a, flags, handle);
}
public static String compare(Set a, Set b, int flags, Map<String, Integer> handle)
{
String cmp;
cmp = compare1(a, b, flags, handle);
if (cmp != null)
return cmp;
return compare1(b, a, flags, handle);
}
private static String compare1(Set a, Set b, int flags, Map<String, Integer> handle) {
for (Object valA : a)
{
int flagsa = flags(handle, valA);
if ((flagsa & SKIP) != 0)
continue;
if (!b.contains(valA))
{
return "[" + valA + "]==null";
}
}
return null;
}
public static String compare(SolrDocument a, SolrDocument b, int flags, Map<String, Integer> handle)
{
return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle);
@@ -321,22 +315,10 @@ public class SolrResponsesComparator
public static String compare(Object[] a, Object[] b, int flags, Map<String, Integer> handle)
{
boolean ordered = (flags & UNORDERED) == 0;
if (a.length != b.length)
{
return ".length:" + a.length + "!=" + b.length;
}
if (!ordered)
{
Set<Object> setA = Sets.newHashSet(a);
Set<Object> setB = Sets.newHashSet(b);
return compare(setA, setB, flags, handle);
}
for (int i = 0; i < a.length; i++)
{
String cmp = compare(a[i], b[i], flags, handle);
@@ -346,7 +328,6 @@ public class SolrResponsesComparator
return null;
}
public static String compare(Object a, Object b, int flags, Map<String, Integer> handle)
{
if (a == b)
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java
+7 -8
View File
@@ -18,8 +18,6 @@
*/
package org.alfresco.solr.tracker;
import static java.util.Arrays.asList;
import static org.alfresco.solr.AlfrescoSolrUtils.ancestors;
import static org.alfresco.solr.AlfrescoSolrUtils.createGUID;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
@@ -350,11 +348,13 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT
logger.info("#################### Passed Fifteenth Test ##############################");
// Change the aclReaders
aclReaders.setReaders(asList("andy"));
List<String> readers = aclReaders.getReaders();
readers.set(0, "andy"); // Change the aclReader
indexAclId(acl.getId());
aclReaders2.setReaders(asList("ice"));
List<String> readers2 = aclReaders2.getReaders();
readers2.set(0, "ice"); // Change the aclReader
reindexAclId(acl2.getId());
@@ -377,9 +377,8 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT
logger.info("#################### Passed Seventeenth Test ##############################");
// Change the aclReaders
aclReaders.setReaders(asList("alan"));
aclReaders2.setReaders(asList("paul"));
readers.set(0, "alan"); // Change the aclReader
readers2.set(0, "paul"); // Change the aclReader
reindexAclChangeSetId(aclChangeSet.getId()); //This should replace "andy" and "ice" with "alan" and "paul"
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java
+7 -8
View File
@@ -18,8 +18,6 @@
*/
package org.alfresco.solr.tracker;
import static java.util.Arrays.asList;
import static org.alfresco.solr.AlfrescoSolrUtils.ancestors;
import static org.alfresco.solr.AlfrescoSolrUtils.createGUID;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
@@ -355,11 +353,13 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT
logger.info("#################### Passed Fifteenth Test ##############################");
// Change the aclReaders
aclReaders.setReaders(asList("andy"));
List<String> readers = aclReaders.getReaders();
readers.set(0, "andy"); // Change the aclReader
indexAclId(acl.getId());
aclReaders2.setReaders(asList("ice"));
List<String> readers2 = aclReaders2.getReaders();
readers2.set(0, "ice"); // Change the aclReader
reindexAclId(acl2.getId());
@@ -382,9 +382,8 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT
logger.info("#################### Passed Seventeenth Test ##############################");
// Change the aclReaders
aclReaders.setReaders(asList("alan"));
aclReaders2.setReaders(asList("paul"));
readers.set(0, "alan"); // Change the aclReader
readers2.set(0, "paul"); // Change the aclReader
reindexAclChangeSetId(aclChangeSet.getId()); //This should replace "andy" and "ice" with "alan" and "paul"
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java
+6 -4
View File
@@ -53,12 +53,14 @@ public class ContentTrackerIT
private TrackerStats trackerStats;
private int UPDATE_BATCH = 2;
private int READ_BATCH = 400;
@Before
public void setUp() throws Exception
{
doReturn("workspace://SpacesStore").when(props).getProperty(eq("alfresco.stores"), anyString());
doReturn("" + UPDATE_BATCH).when(props).getProperty(eq("alfresco.contentUpdateBatchSize"), anyString());
doReturn("" + READ_BATCH).when(props).getProperty(eq("alfresco.contentReadBatchSize"), anyString());
when(srv.getTrackerStats()).thenReturn(trackerStats);
this.contentTracker = new ContentTracker(props, repositoryClient, coreName, srv);
@@ -101,14 +103,14 @@ public class ContentTrackerIT
doc.tenant = "2";
docs2.add(doc);
}
when(this.srv.getDocsWithUncleanContent())
when(this.srv.getDocsWithUncleanContent(anyInt(), anyInt()))
.thenReturn(docs1)
.thenReturn(docs2)
.thenReturn(emptyList);
this.contentTracker.doTrack("anIterationId");
InOrder order = inOrder(srv);
order.verify(srv).getDocsWithUncleanContent();
order.verify(srv).getDocsWithUncleanContent(0, READ_BATCH);
/*
* I had to make each bunch of calls have different parameters to prevent Mockito from incorrectly failing
@@ -129,7 +131,7 @@ public class ContentTrackerIT
order.verify(srv).updateContent(thirdDoc);
order.verify(srv).commit();
order.verify(srv).getDocsWithUncleanContent();
order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH, READ_BATCH);
// From docs2
docRef = new TenantDbId();
@@ -138,7 +140,7 @@ public class ContentTrackerIT
order.verify(srv, times(UPDATE_BATCH)).updateContent(docRef);
order.verify(srv).commit();
order.verify(srv).getDocsWithUncleanContent();
order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH + READ_BATCH, READ_BATCH);
}
@Test
public void typeCheck()
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java
-13
View File
@@ -18,7 +18,6 @@
*/
package org.alfresco.solr.tracker;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.solr.AbstractAlfrescoDistributedIT;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.client.Acl;
@@ -28,9 +27,6 @@ import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.Transaction;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.NamedList;
@@ -46,7 +42,6 @@ import java.util.Properties;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID;
import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
@@ -105,14 +100,6 @@ public class DistributedExpandDbidRangeAlfrescoSolrTrackerIT extends AbstractAlf
bulkAcls,
bulkAclReaders);
//Check for the ACL state stamp.
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID,
bulkAclChangeSet.getId(), bulkAclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST));
BooleanQuery waitForQuery = builder.build();
waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME);
SolrQueryResponse response0 = rangeCheck(0);
NamedList values0 = response0.getValues();
//{start=0,end=100,nodeCount=0,maxDbid=0,density=NaN,expand=0,expanded=false}
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java
-60
View File
@@ -1,60 +0,0 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.tracker;
import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_RANGE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Properties;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.repo.index.shard.ShardMethodEnum;
import org.junit.Test;
/** Unit tests for the {@link DocRouterFactory}. */
public class DocRouterFactoryTest
{
/** Check that a DB_ID_RANGE router can be created. */
@Test
public void testDBIDRANGEWithShardRangeKey()
{
Properties mockProperties = mock(Properties.class);
when(mockProperties.containsKey(SHARD_RANGE_KEY)).thenReturn(true);
when(mockProperties.getProperty(SHARD_RANGE_KEY)).thenReturn("100000000-150000000");
// Call the method under test.
DocRouter docRouter = DocRouterFactory.getRouter(mockProperties, ShardMethodEnum.DB_ID_RANGE);
assertTrue("Expected to get a DBIDRangeRouter.", docRouter instanceof DBIDRangeRouter);
DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter;
assertEquals("Unexpected start of range.", dbidRangeRouter.getStartRange(), 100000000L);
assertEquals("Unexpected end of range.", dbidRangeRouter.getEndRange(), 150000000L);
}
/** Check that an exception is raised if the range information is missing. */
@Test(expected = AlfrescoRuntimeException.class)
public void testDBIDRANGEWithoutShardRangeKey()
{
Properties mockProperties = mock(Properties.class);
DocRouterFactory.getRouter(mockProperties, ShardMethodEnum.DB_ID_RANGE);
}
}
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java
+7 -7
View File
@@ -82,7 +82,7 @@ public class DefaultTrackerPoolFactoryTest
public void testAclDefaultProperties()
{
poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker");
tpe = poolFactory.create();
assertEquals(4, tpe.getCorePoolSize());
assertEquals(4, tpe.getMaximumPoolSize());
@@ -96,7 +96,7 @@ public class DefaultTrackerPoolFactoryTest
properties.put("alfresco.acl.tracker.keepAliveTime", "200");
poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker");
tpe = poolFactory.create();
assertEquals(30, tpe.getCorePoolSize());
assertEquals(40, tpe.getMaximumPoolSize());
assertEquals(200, tpe.getKeepAliveTime(TimeUnit.SECONDS));
@@ -120,7 +120,7 @@ public class DefaultTrackerPoolFactoryTest
properties.put("alfresco.content.tracker.keepAliveTime", "201");
poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "ContentTracker");
tpe = poolFactory.create();
assertEquals(100, tpe.getCorePoolSize());
assertEquals(140, tpe.getMaximumPoolSize());
assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS));
@@ -129,9 +129,9 @@ public class DefaultTrackerPoolFactoryTest
public void testMetaDataDefaultProperties()
{
poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker");
tpe = poolFactory.create();
assertEquals(4, tpe.getCorePoolSize());
assertEquals(4, tpe.getMaximumPoolSize());
assertEquals(120, tpe.getKeepAliveTime(TimeUnit.SECONDS));
@@ -143,9 +143,9 @@ public class DefaultTrackerPoolFactoryTest
properties.put("alfresco.metadata.tracker.maximumPoolSize", "140");
properties.put("alfresco.metadata.tracker.keepAliveTime", "201");
poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker");
tpe = poolFactory.create();
assertEquals(100, tpe.getCorePoolSize());
assertEquals(140, tpe.getMaximumPoolSize());
assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS));
search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java
+9 -10
View File
@@ -18,7 +18,6 @@
*/
package org.alfresco.solr.transformer;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
@@ -82,7 +81,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
{
putHandleDefaults();
//Test 1: Running a simple query without invoking AlfrescoFieldMapperTransformer, expected to see id,DBID and _version_
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts"));
assertNotNull(resp);
SolrDocumentList results = resp.getResults();
assertEquals("Expecting 5 rows",5, results.size());
@@ -108,7 +107,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
putHandleDefaults();
//Test 2: Running simple query with AlfrescoFieldMapperTransformer, expected to see all fields returned
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","*,[fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","*,[fmap]"));
SolrDocument docWithAllFields = resp.getResults().get(0);
assertTrue(docWithAllFields.size() > 3);
@@ -130,7 +129,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
putHandleDefaults();
//Test 3: Running simple query with AlfrescoFieldMapperTransformer, expected to see selected fields returned
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id,DBID,[fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id,DBID,[fmap]"));
assertNotNull(resp);
SolrDocument docWithRequestedFields = resp.getResults().get(0);
@@ -145,7 +144,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
putHandleDefaults();
//Test 4: Running simple query with fmapDocTransformer on non default fields, expected to see selected fields returned
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap]"));
assertNotNull(resp);
SolrDocument docWithRequestedFields3 = resp.getResults().get(0);
@@ -160,7 +159,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
{
putHandleDefaults();
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap], [fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap], [fmap]"));
assertNotNull(resp);
SolrDocument docWithRequestedFields3 = resp.getResults().get(0);
@@ -175,7 +174,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
{
putHandleDefaults();
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_name, score, [fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_name, score, [fmap]"));
assertNotNull(resp);
SolrDocumentList results = resp.getResults();
SolrDocument docWithAllFields = results.get(0);
@@ -190,7 +189,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
{
putHandleDefaults();
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, cm_created, DBID, score, [fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, cm_created, DBID, score, [fmap]"));
assertNotNull(resp);
SolrDocumentList results = resp.getResults();
SolrDocument docWithAllFields = results.get(0);
@@ -206,7 +205,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
{
putHandleDefaults();
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm?title, *name, [fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm?title, *name, [fmap]"));
assertNotNull(resp);
SolrDocumentList results = resp.getResults();
SolrDocument docWithAllFields = results.get(0);
@@ -221,7 +220,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute
{
putHandleDefaults();
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [explain], [fmap]", "sort", "id asc"));
QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [explain], [fmap]"));
assertNotNull(resp);
SolrDocumentList results = resp.getResults();
SolrDocument docWithAllFields = results.get(0);
search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java
+567
View File
@@ -0,0 +1,567 @@
package org.apache.lucene.analysis.minhash;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.minhash.MinHashFilter.FixedSizeTreeSet;
import org.apache.lucene.analysis.minhash.MinHashFilter.LongPair;
import org.apache.lucene.analysis.shingle.ShingleFilterFactory;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
import org.junit.Test;
public class MinHashFilterIT extends BaseTokenStreamTestCase
{
@Test
public void testIntHash() {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(0), 0, 4, 0, hash);
assertEquals(-3485513579396041028L, hash.val1);
assertEquals(6383328099726337777L, hash.val2);
}
@Test
public void testStringHash() throws UnsupportedEncodingException {
LongPair hash = new LongPair();
byte[] bytes = "woof woof woof woof woof".getBytes("UTF-16LE");
MinHashFilter.murmurhash3_x64_128(bytes, 0, bytes.length, 0, hash);
assertEquals(7638079586852243959L, hash.val1);
assertEquals(4378804943379391304L, hash.val2);
}
@Test
public void testSimpleOrder() throws UnsupportedEncodingException {
LongPair hash1 = new LongPair();
hash1.val1 = 1;
hash1.val2 = 2;
LongPair hash2 = new LongPair();
hash2.val1 = 2;
hash2.val2 = 1;
assert (hash1.compareTo(hash2) > 0);
}
@Test
public void testHashOrder() {
assertTrue(!MinHashFilter.isLessThanUnsigned(0l, 0l));
assertTrue(MinHashFilter.isLessThanUnsigned(0l, -1l));
assertTrue(MinHashFilter.isLessThanUnsigned(1l, -1l));
assertTrue(MinHashFilter.isLessThanUnsigned(-2l, -1l));
assertTrue(MinHashFilter.isLessThanUnsigned(1l, 2l));
assertTrue(MinHashFilter.isLessThanUnsigned(Long.MAX_VALUE, Long.MIN_VALUE));
FixedSizeTreeSet<LongPair> minSet = new FixedSizeTreeSet<LongPair>(500);
HashSet<LongPair> unadded = new HashSet<LongPair>();
for (int i = 0; i < 100; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
LongPair peek = null;
if (minSet.size() > 0) {
peek = minSet.last();
}
if (!minSet.add(hash)) {
unadded.add(hash);
} else {
if (peek != null) {
if ((minSet.size() == 500) && !peek.equals(minSet.last())) {
unadded.add(peek);
}
}
}
}
assertEquals(100, minSet.size());
assertEquals(0, unadded.size());
HashSet<LongPair> collisionDetection = new HashSet<LongPair>();
unadded = new HashSet<LongPair>();
minSet = new FixedSizeTreeSet<LongPair>(500);
for (int i = 0; i < 1000000; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
collisionDetection.add(hash);
LongPair peek = null;
if (minSet.size() > 0) {
peek = minSet.last();
}
if (!minSet.add(hash)) {
unadded.add(hash);
} else {
if (peek != null) {
if ((minSet.size() == 500) && !peek.equals(minSet.last())) {
unadded.add(peek);
}
}
}
}
assertEquals(1000000, collisionDetection.size());
assertEquals(500, minSet.size());
assertEquals(999500, unadded.size());
LongPair last = null;
LongPair current = null;
while ((current = minSet.pollLast()) != null) {
if (last != null) {
assertTrue(isLessThan(current, last));
} else {
}
last = current;
}
}
@Test
public void testHashNotRepeated() {
FixedSizeTreeSet<LongPair> minSet = new FixedSizeTreeSet<LongPair>(500);
HashSet<LongPair> unadded = new HashSet<LongPair>();
for (int i = 0; i < 10000; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
LongPair peek = null;
if (minSet.size() > 0) {
peek = minSet.last();
}
if (!minSet.add(hash)) {
unadded.add(hash);
} else {
if (peek != null) {
if ((minSet.size() == 500) && !peek.equals(minSet.last())) {
unadded.add(peek);
}
}
}
}
assertEquals(500, minSet.size());
LongPair last = null;
LongPair current = null;
while ((current = minSet.pollLast()) != null) {
if (last != null) {
assertTrue(isLessThan(current, last));
} else {
}
last = current;
}
}
@Test
public void testMockShingleTokenizer() throws IOException {
Tokenizer mockShingleTokenizer = createMockShingleTokenizer(5,
"woof woof woof woof woof" + " " + "woof woof woof woof puff");
assertTokenStreamContents(mockShingleTokenizer,
new String[] {"woof woof woof woof woof", "woof woof woof woof puff"});
}
@Test
public void testTokenStreamSingleInput() throws IOException {
String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯"};
TokenStream ts = createTokenStream(5, "woof woof woof woof woof", 1, 1, 100, false);
assertTokenStreamContents(ts, hashes, new int[] {0},
new int[] {24}, new String[] {MinHashFilter.MIN_HASH_TYPE}, new int[] {1}, new int[] {1}, 24, 0, null,
true);
ts = createTokenStream(5, "woof woof woof woof woof", 2, 1, 1, false);
assertTokenStreamContents(ts, new String[] {new String(new char[] {0, 0, 8449, 54077, 64133, 32857, 8605, 41409}),
new String(new char[] {0, 1, 16887, 58164, 39536, 14926, 6529, 17276})}, new int[] {0, 0},
new int[] {24, 24}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0}, new int[] {1, 1}, 24, 0, null,
true);
}
@Test
public void testTokenStream1() throws IOException {
String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯",
new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})};
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 1, 1, 100,false);
assertTokenStreamContents(ts, hashes, new int[] {0, 0},
new int[] {49, 49}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0},
new int[] {1, 1}, 49, 0, null, true);
}
private ArrayList<String> getTokens(TokenStream ts) throws IOException {
ArrayList<String> tokens = new ArrayList<String>();
ts.reset();
while (ts.incrementToken()) {
CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class);
String token = new String(termAttribute.buffer(), 0, termAttribute.length());
tokens.add(token);
}
ts.end();
ts.close();
return tokens;
}
private ArrayList<String> getTokens(Analyzer analyzer, String field, String value) throws IOException
{
ArrayList<String> tokens = new ArrayList<String>();
TokenStream ts = analyzer.tokenStream(field, value);
ts.reset();
while(ts.incrementToken())
{
CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class);
String token = new String(termAttribute.buffer(), 0, termAttribute.length());
tokens.add(token);
}
ts.end();
ts.close();
return tokens;
}
@Test
public void testTokenStream2() throws IOException {
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 100, 1, 1, false);
ArrayList<String> tokens = getTokens(ts);
ts.close();
assertEquals(100, tokens.size());
}
@Test
public void testTokenStream3() throws IOException {
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 1, 10, false);
ArrayList<String> tokens = getTokens(ts);
ts.close();
assertEquals(20, tokens.size());
}
@Test
public void testTokenStream4() throws IOException {
TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, false);
ArrayList<String> tokens = getTokens(ts);
ts.close();
assertEquals(20, tokens.size());
ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, true);
tokens = getTokens(ts);
ts.close();
assertEquals(100, tokens.size());
}
@Test
public void testLSHQuery() throws IOException
{
Analyzer analyzer = createMinHashAnalyzer(5, 1, 100);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, config);
Document doc = new Document();
doc.add(new TextField("text", "woof woof woof woof woof", Store.NO));
writer.addDocument(doc);
doc = new Document();
doc.add(new TextField("text", "woof woof woof woof woof puff", Store.NO));
writer.addDocument(doc);
doc = new Document();
doc.add(new TextField("text", "woof woof woof woof puff", Store.NO));
writer.addDocument(doc);
writer.commit();
writer.close();
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", "℁팽徭聙↝ꇁ홱杯"))), Occur.SHOULD);
builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})))), Occur.SHOULD);
builder.setDisableCoord(true);
TopDocs topDocs = searcher.search(builder.build(), 10);
assertEquals(3, topDocs.totalHits);
float score = topDocs.scoreDocs[0].score;
assertEquals(topDocs.scoreDocs[1].score, score/2, 0f);
assertEquals(topDocs.scoreDocs[2].score, score/2, 0f);
}
@Test
public void testLSHQuery2() throws IOException
{
String[] parts = new String[]{"one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"};
int min = 5;
Analyzer analyzer = createMinHashAnalyzer(min, 1, 100);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, config);
for(int i = 0; i < parts.length; i++)
{
StringBuilder builder = new StringBuilder();
for(int j = 0; j < parts.length -i; j++)
{
if(builder.length() > 0)
{
builder.append(" ");
}
builder.append(parts[i+j]);
if(j >= min -1)
{
Document doc = new Document();
doc.add(new TextField("text", builder.toString(), Store.NO));
writer.addDocument(doc);
}
}
}
writer.commit();
writer.close();
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
TopDocs topDocs = searcher.search(buildQuery("text", "one two three four five", min, 1, 100), 100);
assertEquals(6, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "two three four five six", min, 1, 100), 100);
assertEquals(10, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "three four five six seven", min, 1, 100), 100);
assertEquals(12, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "four five six seven eight", min, 1, 100), 100);
assertEquals(12, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "five six seven eight nine", min, 1, 100), 100);
assertEquals(10, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "six seven eight nine ten", min, 1, 100), 100);
assertEquals(6, topDocs.totalHits);
assertAllScores(topDocs, 1.0f);
topDocs = searcher.search(buildQuery("text", "one two three four five six", min, 1, 100), 100);
assertEquals(11, topDocs.totalHits);
topDocs = searcher.search(buildQuery("text", "one two three four five six seven eight nine ten", min, 1, 100), 100);
assertEquals(21, topDocs.totalHits);
for(int i = 0; i < topDocs.totalHits; i++)
{
System.out.println(i+" = "+topDocs.scoreDocs[i]);
}
float topScore = 6.0f;
assertEquals(topDocs.scoreDocs[0].score, topScore, 0.001f);
assertEquals(topDocs.scoreDocs[1].score, topScore * 5/6, 0.001f);
assertEquals(topDocs.scoreDocs[2].score, topScore * 5/6, 0.001f);
assertEquals(topDocs.scoreDocs[3].score, topScore * 4/6, 0.001f);
assertEquals(topDocs.scoreDocs[4].score, topScore * 4/6, 0.001f);
assertEquals(topDocs.scoreDocs[5].score, topScore * 4/6, 0.001f);
assertEquals(topDocs.scoreDocs[6].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[7].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[8].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[9].score, topScore * 3/6, 0.001f);
assertEquals(topDocs.scoreDocs[10].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[11].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[12].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[13].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[14].score, topScore * 2/6, 0.001f);
assertEquals(topDocs.scoreDocs[15].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[16].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[17].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[18].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[19].score, topScore * 1/6, 0.001f);
assertEquals(topDocs.scoreDocs[20].score, topScore * 1/6, 0.001f);
}
private void assertAllScores(TopDocs topDocs, float score)
{
for(int i = 0; i < topDocs.totalHits; i++)
{
assertEquals(topDocs.scoreDocs[i].score, score, 0f);
}
}
private Query buildQuery(String field, String query, int min, int hashCount, int hashSetSize) throws IOException
{
TokenizerChain chain = createMinHashAnalyzer(min, hashCount, hashSetSize);
ArrayList<String> tokens = getTokens(chain, field, query);
chain.close();
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for(String token : tokens)
{
builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", token))), Occur.SHOULD);
}
builder.setDisableCoord(true);
return builder.build();
}
public static TokenStream createTokenStream(int shingleSize, String shingles, int hashCount, int bucketCount, int hashSetSize, boolean withRotation) {
Tokenizer tokenizer = createMockShingleTokenizer(shingleSize, shingles);
HashMap<String,String> lshffargs = new HashMap<String,String>();
lshffargs.put("hashCount", "" + hashCount);
lshffargs.put("bucketCount", "" + bucketCount);
lshffargs.put("hashSetSize", "" + hashSetSize);
lshffargs.put("withRotation", "" + withRotation);
MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs);
return lshff.create(tokenizer);
}
public static TokenizerChain createMinHashAnalyzer(int min, int hashCount, int hashSetSize)
{
WhitespaceTokenizerFactory icutf = new WhitespaceTokenizerFactory(Collections.<String, String>emptyMap());
HashMap<String, String> sffargs = new HashMap<String, String>();
sffargs.put("minShingleSize", ""+min);
sffargs.put("maxShingleSize", ""+min);
sffargs.put("outputUnigrams", "false");
sffargs.put("outputUnigramsIfNoShingles", "false");
sffargs.put("tokenSeparator", " ");
ShingleFilterFactory sff = new ShingleFilterFactory(sffargs);
HashMap<String, String> lshffargs = new HashMap<String, String>();
lshffargs.put("hashCount", ""+hashCount);
lshffargs.put("hashSetSize", ""+hashSetSize);
MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs);
TokenizerChain chain = new TokenizerChain(new CharFilterFactory[]{}, icutf, new TokenFilterFactory[]{sff, lshff});
return chain;
}
public static Tokenizer createMockShingleTokenizer(int shingleSize, String shingles) {
MockTokenizer tokenizer = new MockTokenizer(
new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+([ \t\r\n]+[^ \t\r\n]+){4}").toAutomaton()),
true);
tokenizer.setEnableChecks(true);
if (shingles != null) {
tokenizer.setReader(new StringReader(shingles));
}
return tokenizer;
}
private boolean isLessThan(LongPair hash1, LongPair hash2) {
if (MinHashFilter.isLessThanUnsigned(hash1.val2, hash2.val2)) {
return true;
} else if (hash1.val2 == hash2.val2) {
return (MinHashFilter.isLessThanUnsigned(hash1.val1, hash2.val1));
} else {
return false;
}
}
/**
* An analyzer that uses a tokenizer and a list of token filters to
* create a TokenStream - lifted from SOLR to make this analyzer test lucene only.
*/
public static class TokenizerChain extends Analyzer {
final private CharFilterFactory[] charFilters;
final private TokenizerFactory tokenizer;
final private TokenFilterFactory[] filters;
/**
* Creates a new TokenizerChain.
*
* @param charFilters Factories for the CharFilters to use, if any - if null, will be treated as if empty.
* @param tokenizer Factory for the Tokenizer to use, must not be null.
* @param filters Factories for the TokenFilters to use if any- if null, will be treated as if empty.
*/
public TokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) {
this.charFilters = charFilters;
this.tokenizer = tokenizer;
this.filters = filters;
}
@Override
public Reader initReader(String fieldName, Reader reader) {
if (charFilters != null && charFilters.length > 0) {
Reader cs = reader;
for (CharFilterFactory charFilter : charFilters) {
cs = charFilter.create(cs);
}
reader = cs;
}
return reader;
}
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tk = tokenizer.create();
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
}
return new TokenStreamComponents(tk, ts);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("TokenizerChain(");
for (CharFilterFactory filter: charFilters) {
sb.append(filter);
sb.append(", ");
}
sb.append(tokenizer);
for (TokenFilterFactory filter: filters) {
sb.append(", ");
sb.append(filter);
}
sb.append(')');
return sb.toString();
}
}
}
search-services/alfresco-search/src/test/resources/log4j.xml
+10
View File
@@ -7,6 +7,11 @@
</layout>
</appender>
<!-- Useful tests categories -->
<category name="org.alfresco.solr.AbstractAlfrescoSolrIT">
<priority value="WARN"/>
</category>
<!-- ##### ALL ALFRESCO MESSAGES ##### -->
<!--
<category name="org.alfresco">
@@ -14,6 +19,11 @@
</category>
-->
<!-- ##### SOLR INFORMATION SERVER MESSAGES ##### -->
<category name="org.alfresco.solr.SolrInformationServer">
<priority value="DEBUG"/>
</category>
<!-- ##### ONLY SOLR QUERIES ##### -->
<!--
<category name="org.apache.solr.core.SolrCore.Request">
search-services/alfresco-solrclient-lib/pom.xml
+1 -1
View File
@@ -22,7 +22,7 @@
</distributionManagement>
<properties>
<dependency.alfresco-data-model.version>8.105</dependency.alfresco-data-model.version>
<dependency.alfresco-data-model.version>8.98</dependency.alfresco-data-model.version>
<dependency.jackson.version>2.10.3</dependency.jackson.version>
</properties>
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/BoundedDeque.java
+36 -36
View File
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr;
@@ -38,7 +38,7 @@ public class BoundedDeque<T> implements Iterable<T>
public BoundedDeque(int max)
{
this.max = max;
this.deque = new LinkedBlockingDeque<T>();
setDeque(new LinkedBlockingDeque<T>());
}
/**
@@ -46,21 +46,21 @@ public class BoundedDeque<T> implements Iterable<T>
*/
public int size()
{
return deque.size();
return getDeque().size();
}
public void add(T add)
{
while (deque.size() > (max - 1))
while (getDeque().size() > (max - 1))
{
deque.removeLast();
getDeque().removeLast();
}
deque.addFirst(add);
getDeque().addFirst(add);
}
public T getLast()
{
return deque.getFirst();
return getDeque().getFirst();
}
/*
@@ -70,17 +70,17 @@ public class BoundedDeque<T> implements Iterable<T>
@Override
public Iterator<T> iterator()
{
return deque.iterator();
return getDeque().iterator();
}
/** Get a copy of the deque. */
public LinkedBlockingDeque<T> getDeque()
{
return new LinkedBlockingDeque(deque);
return deque;
}
public void setDeque(LinkedBlockingDeque<T> deque)
{
this.deque = new LinkedBlockingDeque(deque);
this.deque = deque;
}
}
}
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclChangeSets.java
+39 -54
View File
@@ -23,57 +23,42 @@
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.client;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/**
* @author Andy
*
*/
public class AclChangeSets
{
private final List<AclChangeSet> aclChangeSets;
private Long maxChangeSetCommitTime;
private Long maxChangeSetId;
AclChangeSets(List<AclChangeSet> aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId)
{
this.aclChangeSets = (aclChangeSets == null ? null : new ArrayList<>(aclChangeSets));
this.maxChangeSetCommitTime = maxChangeSetCommitTime;
this.maxChangeSetId = maxChangeSetId;
}
public AclChangeSets(List<AclChangeSet> aclChangeSets)
{
this.aclChangeSets = aclChangeSets;
if (!aclChangeSets.isEmpty())
{
this.maxChangeSetCommitTime = aclChangeSets.stream()
.max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs();
this.maxChangeSetId = aclChangeSets.stream()
.max(Comparator.comparing(AclChangeSet::getId)).get().getId();
}
}
public List<AclChangeSet> getAclChangeSets()
{
return Collections.unmodifiableList(aclChangeSets);
}
public Long getMaxChangeSetCommitTime()
{
return maxChangeSetCommitTime;
}
public Long getMaxChangeSetId()
{
return maxChangeSetId;
}
}
package org.alfresco.solr.client;
import java.util.List;
/**
* @author Andy
*
*/
public class AclChangeSets
{
private List<AclChangeSet> aclChangeSets;
private Long maxChangeSetCommitTime;
private Long maxChangeSetId;
AclChangeSets(List<AclChangeSet> aclChangeSets, Long maxChangeSetCommitTime, Long maxChangeSetId)
{
this.aclChangeSets = aclChangeSets;
this.maxChangeSetCommitTime = maxChangeSetCommitTime;
this.maxChangeSetId = maxChangeSetId;
}
public List<AclChangeSet> getAclChangeSets()
{
return aclChangeSets;
}
public Long getMaxChangeSetCommitTime()
{
return maxChangeSetCommitTime;
}
public Long getMaxChangeSetId()
{
return maxChangeSetId;
}
}
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AclReaders.java
+30 -38
View File
@@ -1,33 +1,30 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.client;
import static java.util.Collections.unmodifiableList;
import java.util.ArrayList;
import java.util.List;
/**
@@ -39,7 +36,7 @@ public class AclReaders
{
private final long id;
private List<String> readers;
private final List<String> readers;
private final List<String> denied;
@@ -50,8 +47,8 @@ public class AclReaders
public AclReaders(long id, List<String> readers, List<String> denied, long aclChangeSetId, String tenantDomain)
{
this.id = id;
this.readers = (readers == null ? null : new ArrayList<>(readers));
this.denied = (denied == null ? null : new ArrayList<>(denied));
this.readers = readers;
this.denied = denied;
this.aclChangeSetId = aclChangeSetId;
this.tenantDomain = tenantDomain;
}
@@ -99,17 +96,12 @@ public class AclReaders
public List<String> getReaders()
{
return unmodifiableList(readers);
}
public void setReaders(List readers)
{
this.readers = new ArrayList<>(readers);
return readers;
}
public List<String> getDenied()
{
return unmodifiableList(denied);
return denied;
}
public long getAclChangeSetId()
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModel.java
+27 -29
View File
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.client;
import org.alfresco.repo.dictionary.M2Model;
@@ -52,8 +52,7 @@ public class AlfrescoModel
{
return checksum;
}
@Override
public boolean equals(Object other)
{
if (this == other)
@@ -71,8 +70,7 @@ public class AlfrescoModel
checksum.equals(model.getChecksum()));
}
@Override
public int hashCode()
public int hashcode()
{
int result = 17;
result = 31 * result + model.hashCode();
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/AlfrescoModelDiff.java
+30 -32
View File
@@ -1,28 +1,28 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.client;
import org.alfresco.service.namespace.QName;
@@ -36,10 +36,10 @@ import org.alfresco.service.namespace.QName;
*/
public class AlfrescoModelDiff
{
public enum TYPE
public static enum TYPE
{
NEW, CHANGED, REMOVED;
}
};
private QName modelName;
private TYPE type;
@@ -74,8 +74,7 @@ public class AlfrescoModelDiff
{
return newChecksum;
}
@Override
public boolean equals(Object other)
{
if(this == other)
@@ -92,9 +91,8 @@ public class AlfrescoModelDiff
diff.getOldChecksum().equals(getOldChecksum()) &&
diff.getNewChecksum().equals(getNewChecksum()));
}
@Override
public int hashCode()
public int hashcode()
{
int result = 17;
result = 31 * result + getModelName().hashCode();
search-services/alfresco-solrclient-lib/src/main/java/org/alfresco/solr/client/Transactions.java
+1 -14
View File
@@ -25,7 +25,6 @@
*/
package org.alfresco.solr.client;
import java.util.Comparator;
import java.util.List;
/**
@@ -45,19 +44,7 @@ public class Transactions
this.transactions = transactions;
this.maxTxnCommitTime = maxTxnCommitTime;
this.maxTxnId = maxTxnId;
}
public Transactions(List<Transaction> transactions)
{
this.transactions = transactions;
if (!transactions.isEmpty())
{
this.maxTxnCommitTime = transactions.stream()
.max(Comparator.comparing(Transaction::getCommitTimeMs)).get().getCommitTimeMs();
this.maxTxnId = transactions.stream()
.max(Comparator.comparing(Transaction::getId)).get().getId();
}
}
}
public List<Transaction> getTransactions()
{
search-services/alfresco-solrclient-lib/src/test/java/org/alfresco/solr/BoundedDequeTest.java
-57
View File
@@ -1,57 +0,0 @@
/*
* #%L
* Alfresco Solr Client
* %%
* Copyright (C) 2005 - 2016 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.concurrent.LinkedBlockingDeque;
import org.junit.Test;
/** Unit tests for the {@link BoundedDeque}. */
public class BoundedDequeTest
{
/** Check that earlier entries are removed from the BoundedDeque. */
@Test
public void testBoundedness()
{
// Create a BoundedDeque with size two.
BoundedDeque<Object> boundedDeque = new BoundedDeque<>(2);
// Add three things.
boundedDeque.add("A");
boundedDeque.add("B");
boundedDeque.add("C");
// Check that the latest two are still there.
LinkedBlockingDeque<Object> actual = boundedDeque.getDeque();
assertEquals("Unexpected entries in BoundedDeque.", asList("C", "B"), new ArrayList(actual));
}
}
search-services/packaging/src/docker/6.x/docker-compose.yml
-2
View File
@@ -70,8 +70,6 @@ services:
- ALFRESCO_SECURE_COMMS=none
#Enable Spellcheck by setting to true
- ENABLE_SPELLCHECK=${SEARCH_ENABLE_SPELLCHECK}
#Disable cascade tracking by setting to true, default false - as cascade tracking is enabled.
- DISABLE_CASCADE_TRACKING=${DISABLE_CASCADE_TRACKING}
ports:
- 8083:8983 #Browser port
search-services/packaging/src/docker/Dockerfile
+1 -1
View File
@@ -1,6 +1,6 @@
# Alfresco Search Services ${project.version} Docker Image
FROM alfresco/alfresco-base-java:11.0.7-openjdk-centos-7-145f5d28c8ca
FROM alfresco/alfresco-base-java:11.0.1-openjdk-centos-7-72b88c6f1f4c
LABEL creator="Gethin James" maintainer="Alfresco Search Services Team"
ENV DIST_DIR /opt/alfresco-search-services
search-services/packaging/src/docker/search_config_setup.sh
-4
View File
@@ -101,8 +101,4 @@ if [[ true == "$ENABLE_SPELLCHECK" ]]; then
sed -i 's/#alfresco.suggestable.property/alfresco.suggestable.property/' ${PWD}/solrhome/conf/shared.properties
fi
if [[ true == "$DISABLE_CASCADE_TRACKING" ]]; then
sed -i 's/alfresco.cascade.tracker.enabled=true/alfresco.cascade.tracker.enabled=false/' ${PWD}/solrhome/conf/shared.properties
fi
bash -c "$@"