diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index c7ea6717b..3170f8cfb 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -11,7 +11,7 @@ Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share - 1.38 + 1.37 1.13 3.0.20 3.3.0.1 diff --git a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java index 5155b8bd5..8da8c14e8 100644 --- a/e2e-test/src/main/java/org/alfresco/search/TestGroup.java +++ b/e2e-test/src/main/java/org/alfresco/search/TestGroup.java @@ -34,8 +34,6 @@ public class TestGroup public static final String CONFIG_SHARDING_EXPLICIT ="CONFIG_Sharding_EXPLICIT"; // Alfresco search services using sharded environment and explicit routing public static final String CONFIG_SHARDING_DB_ID_RANGE = "CONFIG_Sharding_DB_ID_RANGE"; // Alfresco Search Services using Sharding with DB_ID_RANGE - public static final String CONFIG_ENABLED_CASCADE_TRACKER ="Config_Enabled_Cascade_Tracker"; // Alfresco search services does not index fields related to cascaded updates - public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n diff --git a/e2e-test/src/main/resources/model/music-model.xml b/e2e-test/src/main/resources/model/music-model.xml index d7de00d47..9406e7e78 100644 --- a/e2e-test/src/main/resources/model/music-model.xml +++ b/e2e-test/src/main/resources/model/music-model.xml @@ -66,7 +66,7 @@ d:text false - BOTH + TRUE false diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java index 0fe47d474..5228c38e6 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java @@ -1,6 +1,5 @@ package org.alfresco.test.search.functional.searchServices.cmis; -import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.data.provider.XMLDataConfig; @@ -140,16 +139,4 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest cmisApi.authenticateUser(testUser); Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } - - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @XMLDataConfig(file = "src/test/resources/testdata/search-by-aspect-in-tree.xml") - public void executeSearchByAspectInTree(QueryModel query) throws Exception - { - String currentQuery = query.getValue() - .replace("NODE_REF[f1]", tasFolder1.getNodeRef()) - .replace("NODE_REF[s1]", siteDoclibNodeRef); - - cmisApi.authenticateUser(testUser); - Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); - } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java index 3a52c75dd..8d10955fd 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java @@ -1,6 +1,5 @@ package org.alfresco.test.search.functional.searchServices.cmis; -import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.data.provider.XMLDataConfig; @@ -101,19 +100,4 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest cmisApi.authenticateUser(testUser); Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); } - - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @XMLDataConfig(file = "src/test/resources/testdata/search-by-id-in-tree.xml") - public void executeSearchByIdInTree(QueryModel query) throws Exception - { - String currentQuery = query.getValue() - .replace("NODE_REF[siteId]", siteDoclibNodeRef) - .replace("NODE_REF[d1]", tasSubFile1.getNodeRefWithoutVersion()) - .replace("NODE_REF[d2]", tasSubFile2.getNodeRefWithoutVersion()) - .replace("NODE_REF[f1]", tasFolder1.getNodeRef()) - .replace("NODE_REF[f1-1]", tasSubFolder1.getNodeRef()); - - cmisApi.authenticateUser(testUser); - Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery)); - } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java index a9dc4e4a7..1c86b2912 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java @@ -1,6 +1,5 @@ package org.alfresco.test.search.functional.searchServices.cmis; -import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.provider.XMLTestData; @@ -49,7 +48,7 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest Utility.waitToLoopTime(getSolrWaitTimeInSeconds()); } - @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test(dependsOnMethods = "prepareDataForSearchByPath", dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") @XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml") public void executeSearchByPathQueries(QueryModel query) { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java index 1be2a268e..cf6647e86 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java @@ -1,6 +1,5 @@ package org.alfresco.test.search.functional.searchServices.cmis; -import org.alfresco.search.TestGroup; import org.alfresco.utility.Utility; import org.alfresco.utility.data.provider.XMLDataConfig; import org.alfresco.utility.data.provider.XMLTestDataProvider; @@ -56,7 +55,7 @@ public class SolrSearchInTreeTests extends AbstractCmisE2ETest dataContent.deleteSite(testSite); } - @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData", groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData") @XMLDataConfig(file = "src/test/resources/testdata/search-in-tree.xml") public void executeCMISQuery(QueryModel query) throws Exception { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java index f6bf50849..50e75cc67 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetFieldsSearchTest.java @@ -31,10 +31,8 @@ import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.RestResultBucketsModel; import org.alfresco.rest.search.SearchRequest; import org.alfresco.rest.search.SearchResponse; -import org.alfresco.search.TestGroup; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.data.RandomData; -import org.alfresco.utility.data.provider.XMLTestDataProvider; import org.alfresco.utility.model.FileModel; import org.alfresco.utility.model.FileType; import org.alfresco.utility.model.FolderModel; @@ -95,7 +93,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest waitForContentIndexing(htmlFile.getContent(), true); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test public void testSearchFacetFieldsBucketExcludedWhenMinCount2() throws Exception { // Create Query with FacetFields: Site and Content MimeType @@ -131,7 +129,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test public void testSearchWithFacetFieldsMinCountChecks() throws Exception { SearchRequest query = new SearchRequest(); @@ -206,7 +204,7 @@ public class FacetFieldsSearchTest extends AbstractE2EFunctionalTest Assert.assertNull(response.getContext().getFacetsFields()); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test public void testSearchWithFacetFieldsOnlyFacetsWhereAccess() throws Exception { SearchRequest query = new SearchRequest(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java index 8614deb4d..5b52698a2 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetRangeSearchTest.java @@ -109,8 +109,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest .containsSummary(String.format(RestErrorModel.MANDATORY_PARAM, "gap")); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, + @Test + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") public void searchWithRange() @@ -159,8 +159,8 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest assertEquals(info.get("endInclusive"), "true"); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, + @Test + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Check date facet intervals search api") @SuppressWarnings("unchecked") public void searchWithRangeHardend() @@ -257,7 +257,7 @@ public class FacetRangeSearchTest extends AbstractSearchServicesE2ETest query.setRanges(ranges); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Check basic facet range search api") @SuppressWarnings("unchecked") diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java index 0da2b5ebc..ce389980d 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FacetedSearchTest.java @@ -102,8 +102,8 @@ public class FacetedSearchTest extends AbstractSearchServicesE2ETest waitForContentIndexing(file4.getContent(), true); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") + @Test + @TestRail(section = { TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Checks facet queries for the Search api") public void searchWithQueryFaceting() throws Exception { SearchRequest query = new SearchRequest(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java index ac07b7718..858e8b2f5 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/FingerPrintTest.java @@ -18,11 +18,9 @@ */ package org.alfresco.test.search.functional.searchServices.search; -import static jersey.repackaged.com.google.common.collect.Sets.newHashSet; import static org.testng.Assert.assertTrue; -import java.util.Set; - +import org.alfresco.rest.search.SearchNodeModel; import org.alfresco.rest.search.SearchResponse; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.model.FileModel; @@ -37,18 +35,12 @@ import org.testng.annotations.Test; * Search end point Public API test with finger print. * * @author Michael Suzuki - * @author Meenal Bhave */ public class FingerPrintTest extends AbstractE2EFunctionalTest { + private FileModel file1; + private FileModel file2; private FolderModel folder; - private FileModel fileBanana, fileTaco, fileCat, fileDog, fileOriginal; - /** - * File to be created in dataprep with same content as fileOriginal - * File to be updated in test searchAfterVersionUpdate - * File to be reverted in test searchAfterVersionRevert - */ - private FileModel fileToBeUpdated; @BeforeClass(alwaysRun = true) public void dataPreparation() throws Exception @@ -59,39 +51,30 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest * |-- pangram-banana.txt * |-- pangram-taco.txt * |-- pangram-cat.txt - * |-- pangram-dog.txt - * |-- original.txt - * |-- toBeUpdated.txt + * |-- dog.txt */ folder = new FolderModel("The quick brown fox jumps over"); dataContent.usingUser(testUser).usingSite(testSite).createFolder(folder); - fileBanana = new FileModel("pangram-banana.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy banana"); - fileTaco = new FileModel("pangram-taco.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy dog that ate the taco"); + file1 = new FileModel("pangram-banana.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy banana"); + file2 = new FileModel("pangram-taco.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy dog that ate the taco"); - fileCat = new FileModel("pangram-cat.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy cat"); - fileDog = new FileModel("pangram-dog.txt", FileType.TEXT_PLAIN, "The quick brown fox ate the lazy dog"); - - // fileOriginal with different content - fileOriginal = new FileModel("original.txt", FileType.TEXT_PLAIN, "This is an original file with some content"); - - // Index a new fileToBeUpdated with content similar to fileOriginal - fileToBeUpdated = new FileModel("toBeUpdated.txt", FileType.TEXT_PLAIN, "This is an original file with some content"); + FileModel file3 = new FileModel("pangram-cat.txt", FileType.TEXT_PLAIN, "The quick brown fox jumps over the lazy cat"); + FileModel file4 = new FileModel("dog.txt", FileType.TEXT_PLAIN, "The quick brown fox ate the lazy dog"); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileBanana); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileTaco); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileCat); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileDog); - - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileOriginal); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(fileToBeUpdated); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file1); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file2); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file3); + dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder).createContent(file4); + + waitForContentIndexing(file4.getContent(), true); // Additional wait implemented to remove inconsistent failures. Ref: Search-1438 for details - waitForIndexing("FINGERPRINT:" + fileDog.getNodeRefWithoutVersion(), true); - waitForIndexing("FINGERPRINT:" + fileToBeUpdated.getNodeRefWithoutVersion(), true); + waitForIndexing("FINGERPRINT:" + file2.getNodeRefWithoutVersion(), true); + waitForIndexing("FINGERPRINT:" + file3.getNodeRefWithoutVersion(), true); } - @Test(priority = 1) + @Test @Bug(id = "MNT-20449") public void makeSureFingerprintQueryWorksAfterMetadataUpdate() throws Exception { @@ -122,125 +105,86 @@ public class FingerPrintTest extends AbstractE2EFunctionalTest * to the files loaded as part of this test. * Note that for fingerprint to work it need a 5 word sequence. */ - @Test(priority = 2) + @Test public void search() { - String uuid = fileBanana.getNodeRefWithoutVersion(); + String uuid = file1.getNodeRefWithoutVersion(); Assert.assertNotNull(uuid); - - String fingerprintQuery = String.format("FINGERPRINT:%s", uuid); - SearchResponse response = queryAsUser(testUser, fingerprintQuery); - + String fingerprint = String.format("FINGERPRINT:%s", uuid); + SearchResponse response = query(fingerprint); int count = response.getEntries().size(); assertTrue(count > 1); - - Set expectedNames = newHashSet(); - expectedNames.add(fileBanana.getName()); - expectedNames.add(fileTaco.getName()); - expectedNames.add(fileCat.getName()); - - testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS); + for (SearchNodeModel m : response.getEntries()) + { + String match = m.getModel().getName(); + switch (match) + { + case "pangram.txt": + case "pangram-banana.txt": + case "pangram-taco.txt": + case "pangram-cat.txt": + break; + default: + throw new AssertionError("Not a match to an expected file: " + m.getModel().getName()); + } + m.getModel().assertThat().field("name").isNot("dog.txt"); + m.getModel().assertThat().field("name").isNot("cars.txt"); + } } - @Test(priority = 3) + @Test public void searchSimilar() { - String uuid = fileTaco.getNodeRefWithoutVersion(); + String uuid = file2.getNodeRefWithoutVersion(); Assert.assertNotNull(uuid); // In the response entity there is a score of each doc, change below threshold to bring more like or less. - String fingerprintQuery = String.format("FINGERPRINT:%s_68", uuid); - SearchResponse response = query(fingerprintQuery); + String fingerprint = String.format("FINGERPRINT:%s_68", uuid); + SearchResponse response = query(fingerprint); int count = response.getEntries().size(); assertTrue(count > 1); - - Set expectedNames = newHashSet(); - expectedNames.add(fileTaco.getName()); - - testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS); + + for (SearchNodeModel m : response.getEntries()) + { + switch (m.getModel().getName()) + { + case "pangram.txt": + case "pangram-taco.txt": + break; + default: + throw new AssertionError("Not a match to an expected file: " + m.getModel().getName()); + } + m.getModel().assertThat().field("name").isNot("pangram-banana.txt"); + m.getModel().assertThat().field("name").isNot("pangram-cat.txt"); + m.getModel().assertThat().field("name").isNot("dog.txt"); + m.getModel().assertThat().field("name").isNot("cars.txt"); + } } - @Test(priority = 4) + @Test public void searchSimilar67Percent() { - String uuid = fileTaco.getNodeRefWithoutVersion(); + String uuid = file2.getNodeRefWithoutVersion(); Assert.assertNotNull(uuid); - - // Check that: When asked for less overlap of 67%: More documents are matched - String fingerprintQuery = String.format("FINGERPRINT:%s_67", uuid); - SearchResponse response = query(fingerprintQuery); - + String fingerprint = String.format("FINGERPRINT:%s_68", uuid); + SearchResponse response = query(fingerprint); int count = response.getEntries().size(); assertTrue(count > 1); - - Set expectedNames = newHashSet(); - expectedNames.add(fileTaco.getName()); - expectedNames.add(fileBanana.getName()); - expectedNames.add(fileCat.getName()); - - testSearchQueryUnordered(fingerprintQuery, expectedNames, SearchLanguage.AFTS); - } - - @Test(priority = 5, enabled = false) - @Bug(id = "SEARCH-2065") - public void searchAfterVersionUpdate() - { - // Check that fileToBeUpdated is found with a fingerprint query with fileOriginal: as they have similar content - boolean found = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); - Assert.assertTrue(found, "Matching File Not found in results for Fingerprint Query with original file"); - - // Update content of the fileToBeUpdated to match file1 in the dataprep - String newFileContent = "The quick brown fox jumps over the updated file"; - fileToBeUpdated.setContent(newFileContent); - dataContent.usingUser(testUser).usingSite(testSite).usingResource(fileToBeUpdated).updateContent(newFileContent); - - // Wair for the new version of the file to be indexed - assertTrue(waitForContentIndexing(fileToBeUpdated.getContent(), true)); - - // Check that fileToBeUpdated is NOT found with a fingerprint query with fileOriginal - boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false); - Assert.assertTrue(notFound, "Updated File unexpectedly found in results for Fingerprint Query with original file"); - - // Check that fileToBeUpdated is found with a fingerprint query with file1 - found = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); - Assert.assertTrue(found, "Update File Not found in results for Fingerprint Query with updated content"); - } - - @Test(priority = 6, enabled = false) - @Bug(id = "SEARCH-2065") - public void searchAfterVersionRevert() throws Exception - { - // Revert fileToBeUpdated to previous version - restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).revertVersion("1.0", "{}"); - String revertedContent = restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).getVersionContent("1.2").toString(); - Assert.assertEquals(revertedContent, fileOriginal.getContent(), "Reverted content does not match Original"); - - // Wair for the new version of the file to be indexed - assertTrue(waitForContentIndexing(fileToBeUpdated.getContent(), true)); - - // Check that fileToBeUpdated is found with a fingerprint query with fileOriginal - boolean found = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); - Assert.assertTrue(found, "File not found in results for Fingerprint Query with original file after reverting version changes"); - - // Check that fileToBeUpdated is NOT found with a fingerprint query with file1 - boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false); - Assert.assertTrue(notFound, "File appears in the results for Fingerprint Query even after reverting content changes"); - } - - @Test(priority = 7, enabled = false) - @Bug(id = "SEARCH-2065") - public void searchAfterVersionDelete() throws Exception - { - // Revert fileToBeUpdated to previous version - restClient.authenticateUser(testUser).withCoreAPI().usingNode(fileToBeUpdated).deleteNodeVersion("1.2"); - - // Check that fileToBeUpdated is NOT found with a fingerprint query with fileOriginal - boolean notFound = isContentInSearchResults("FINGERPRINT:" + fileOriginal.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), false); - Assert.assertTrue(notFound, "Updated File unexpectedly found in results for Fingerprint Query with original file"); - - // Check that fileToBeUpdated is found with a fingerprint query with file1 - boolean found = isContentInSearchResults("FINGERPRINT:" + fileBanana.getNodeRefWithoutVersion(), fileToBeUpdated.getName(), true); - Assert.assertTrue(found, "Update File Not found in results for Fingerprint Query with updated content"); + for (SearchNodeModel m : response.getEntries()) + { + switch (m.getModel().getName()) + { + case "pangram.txt": + case "pangram-taco.txt": + case "pangram-cat.txt": + break; + default: + throw new AssertionError("Not a match to an expected file: " + m.getModel().getName()); + } + m.getModel().assertThat().field("name").isNot("pangram-banana.txt"); + m.getModel().assertThat().field("name").isNot("dog.txt"); + m.getModel().assertThat().field("name").isNot("cars.txt"); + } } } diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java index 8bba9f7b0..c99a89ae4 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/PivotFacetedSearchTest.java @@ -119,8 +119,8 @@ public class PivotFacetedSearchTest extends AbstractSearchServicesE2ETest assertPivotResponse(response, "creator", null); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, + @Test + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Checks nested pivot using Search api") public void searchWithNestedPivoting() diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java index 1ce552969..c95bd5a87 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java @@ -155,7 +155,7 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest }); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test(priority = 4) public void testSearchInFieldSITE() { String query = "SITE:" + testSite.getId(); diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java index 95df381d7..1016869cc 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAPATHTest.java @@ -29,7 +29,6 @@ import org.alfresco.rest.search.RestRequestQueryModel; import org.alfresco.rest.search.RestResultBucketsModel; import org.alfresco.rest.search.SearchRequest; import org.alfresco.rest.search.SearchResponse; -import org.alfresco.search.TestGroup; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -93,7 +92,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest *}} * */ - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test public void searchLevel0() { SearchRequest searchQuery = searchRequestWithAPATHFacet("name:*", "0"); @@ -109,7 +108,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest * Test to test that the facet buckets are returned correctly for sub-level 1/ * Test to search for a searchString, that's unique to the test run and hence stable for any environment */ - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test public void searchLevel0andIncludeSubLevel1() { SearchRequest searchQuery = searchRequestWithAPATHFacet("name:" + unique_searchString, "1/"); @@ -121,7 +120,7 @@ public class SearchAPATHTest extends AbstractSearchServicesE2ETest getFirstBucket(response).assertThat().field("label").contains("1/"); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test public void searchLevel2() { String queryString = "name:cars"; diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java index c9e0e0dcf..b382bc6f3 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSecondaryAssociationTest.java @@ -9,7 +9,6 @@ package org.alfresco.test.search.functional.searchServices.search; import org.alfresco.rest.model.RestNodeAssociationModelCollection; import org.alfresco.rest.model.RestNodeChildAssociationModel; -import org.alfresco.search.TestGroup; import org.alfresco.test.search.functional.AbstractE2EFunctionalTest; import org.alfresco.utility.data.CustomObjectTypeProperties; import org.alfresco.utility.model.FileModel; @@ -53,7 +52,7 @@ public class SearchSecondaryAssociationTest extends AbstractE2EFunctionalTest waitForMetadataIndexing(file1.getName(), true); } - @Test(priority = 1, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test(priority = 1) public void testSearchPathForSecondaryAssociation() throws Exception { String queryPathFolder1 = "PATH:\"/app:company_home/st:sites/cm:" + testSite.getTitle() + diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java index 72c6ab684..aa49cf8e5 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchTest.java @@ -101,8 +101,8 @@ public class SearchTest extends AbstractSearchServicesE2ETest response.getContext().assertThat().field("request").isNotEmpty(); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, + @Test + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithOneSortClause() { @@ -148,8 +148,8 @@ public class SearchTest extends AbstractSearchServicesE2ETest * The first clause has always the same value for all matches so the test makes sure the request is correctly * processed and the returned order is determined by the second clause. */ - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) - @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, + @Test + @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH }, executionType = ExecutionType.REGRESSION, description = "Tests a search request containing a sort clause.") public void searchWithTwoSortClauses() { diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java index 0236fb1ed..5de017a74 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java @@ -203,7 +203,7 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest assertTrue((Integer)metricCount.get("countValues") > count, "With the exclude filter there will be more documents than returned"); } - @Test(groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test @TestRail(section = {TestGroup.REST_API, TestGroup.SEARCH}, executionType = ExecutionType.REGRESSION, description = "Checks errors with stats with Pivot using Search api") diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java index 84570427f..75b601800 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java @@ -76,7 +76,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest "failing while removing sharding model"); } - @Test(priority = 1, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test(priority = 1) public void testChildPathWhenParentRenamed() throws Exception { // Create Parent folder @@ -119,7 +119,7 @@ public class CascadingIntegrationTest extends AbstractE2EFunctionalTest Assert.assertEquals(descendantCountOfOriginalName, 0, "Old path still has descendants: " + parentQuery); } - @Test(priority = 2, groups={TestGroup.CONFIG_ENABLED_CASCADE_TRACKER}) + @Test(priority = 2) public void testGrandChildPathWhenGrandParentRenamed() throws Exception { // Create grand parent folder diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java index 20666ad8c..52cbc65f9 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java @@ -475,24 +475,17 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest checkResponseStatusOk(response); - Integer expand = response.getResponse().body().jsonPath().get("expand"); + Integer expand = response.getResponse().body().jsonPath().get("expand"); // RangeCheck action only applies to DB_ID_RANGE Sharding method, so expect error in other sharding methods and success for DB_ID_RANGE if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod())) { // This assertion replicates: testRangeCheckSharding, priority = 21, hence deleting that test as duplicate - // Value -1 is expected when the next shard already has nodes indexed. - Assert.assertTrue((expand == Integer.valueOf(-1) || expand == Integer.valueOf(0)), "RangeCheck should not have been allowed when not using Shard DB_ID_RANGE method,"); - - Integer minDbid = response.getResponse().body().jsonPath().get("minDbid"); - Assert.assertTrue(minDbid > 0, "RangeCheck is not successful when not using Shard DB_ID_RANGE method,"); + Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); } else { - Assert.assertEquals(expand, Integer.valueOf(-1), "RangeCheck should not have been allowed when not using Shard DB_ID_RANGE method,"); - - String exception = response.getResponse().body().jsonPath().get("exception"); - Assert.assertEquals(exception, "ERROR: Wrong document router type:DBIDRouter", "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,"); + Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,"); } } @@ -516,18 +509,12 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest // Expand action only applies to DB_ID_RANGE Sharding method, so expect error in other sharding methods and success for DB_ID_RANGE if (ShardingMethod.DB_ID_RANGE.toString().equalsIgnoreCase(getShardMethod())) { - // This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate - Assert.assertTrue((expand == Integer.valueOf(-1) || expand == Integer.valueOf(0)), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); - if (expand == Integer.valueOf(-1)) - { - String exceptionExpected = "Expansion cannot occur if max DBID in the index is more then 75% of range."; - String exception = response.getResponse().body().jsonPath().get("exception"); - Assert.assertEquals(exception, exceptionExpected, "Expansion failed with unexpected Exception while using DB_ID_RANGE sharding"); - } + // This assertion replicates: testExpandSharding, priority = 23, hence deleting that test as duplicate + Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is not successful when not using Shard DB_ID_RANGE method,"); } else { - Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method"); + Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion should not have been allowed when not using Shard DB_ID_RANGE method,"); } } diff --git a/e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml b/e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml deleted file mode 100644 index 2f718ccaf..000000000 --- a/e2e-test/src/test/resources/testdata/search-by-aspect-in-tree.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/e2e-test/src/test/resources/testdata/search-by-aspect.xml b/e2e-test/src/test/resources/testdata/search-by-aspect.xml index cd47c7a92..ef7090d59 100644 --- a/e2e-test/src/test/resources/testdata/search-by-aspect.xml +++ b/e2e-test/src/test/resources/testdata/search-by-aspect.xml @@ -1,5 +1,6 @@ + @@ -57,6 +58,27 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml b/e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml deleted file mode 100644 index b3d17d088..000000000 --- a/e2e-test/src/test/resources/testdata/search-by-id-in-tree.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/e2e-test/src/test/resources/testdata/search-by-id.xml b/e2e-test/src/test/resources/testdata/search-by-id.xml index a34c8463f..9c6cf645e 100644 --- a/e2e-test/src/test/resources/testdata/search-by-id.xml +++ b/e2e-test/src/test/resources/testdata/search-by-id.xml @@ -6,6 +6,7 @@ + diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index b6c574b25..0859c1ecc 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -18,8 +18,6 @@ */ package org.alfresco.solr; -import static java.util.Collections.unmodifiableList; - import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -796,7 +794,8 @@ public class AlfrescoSolrDataModel implements QueryConstants fieldName = getStoredTextField(propertyName); } - indexedField.addField(fieldName, false, false); + FieldInstance field = new FieldInstance(fieldName, false, false); + indexedField.getFields().add(field); } /* @@ -1437,7 +1436,7 @@ public class AlfrescoSolrDataModel implements QueryConstants public List getFields() { - return unmodifiableList(fields); + return fields; } public void addField(String prefix, boolean localised, boolean sort) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java index 9d8052652..aedad61fe 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerOfResources.java @@ -40,7 +40,7 @@ import org.apache.solr.common.params.SolrParams; */ public class HandlerOfResources { - protected static final List DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.", + public static final List DISALLOWED_SHARED_UPDATES = Arrays.asList("alfresco.identifier.property.", "alfresco.suggestable.property.", "alfresco.cross.locale.property.", "alfresco.cross.locale.datatype."); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java index 16072939e..82a91b6ee 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java @@ -148,7 +148,7 @@ public interface InformationServer extends InformationServerCollectionProvider IndexHealthReport reportIndexTransactions(Long minTxId, IOpenBitSet txIdsInDb, long maxTxId) throws IOException; - List getDocsWithUncleanContent() throws IOException; + List getDocsWithUncleanContent(int start, int rows) throws IOException; void updateContent(TenantDbId docRef) throws Exception; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java new file mode 100644 index 000000000..f28ce84ce --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/ResizeableArrayList.java @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2005-2013 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +import java.io.Serializable; +import java.util.AbstractList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.RandomAccess; + +/** + * A {@link List} implementation, backed by an array, taht supports resizing. + * + * This class supports the reuse of arrays across cache instances, reducing the number of + * temporary objects created by the Alfresco Solr indexing service. + * + * @author Alex Miller + */ +public class ResizeableArrayList extends AbstractList implements List, RandomAccess, Cloneable, Serializable +{ + private static final long serialVersionUID = 1L; + + private Object[] values; + private int size; + boolean active = false; + + public ResizeableArrayList() + { + this(10); + } + + public ResizeableArrayList(int initialSize) + { + values = new Object[initialSize]; + size = initialSize; + } + + @SuppressWarnings("unchecked") + @Override + public E get(int index) + { + checkSize(index); + return (E)values[index]; + } + + private void checkSize(int index) + { + if (index >= size) + { + throw new ArrayIndexOutOfBoundsException("Index: " + index + ", Size: " + values.length); + } + } + + /** + * Resize the underlying to at least minSize. + * + * @param minSize + * @throws IllegalStateException if the instance is not active. + */ + public void resize(int minSize) + { + isActive(); + int oldSize = values.length; + if (minSize > oldSize) + { + int newSize = (oldSize * 3)/2 + 1; + if (newSize < minSize) + { + newSize = minSize; + } + // minCapacity is usually close to size, so this is a win: + values = Arrays.copyOf(values, newSize); + } + size = minSize; + } + + @Override + public int size() + { + return size; + } + + /** + * Copy elements from from into this instance + */ + public void copyFrom(ResizeableArrayList from) + { + isActive(); + values = Arrays.copyOf(from.values, from.values.length); + size = from.size; + } + + @SuppressWarnings("unchecked") + @Override + public E set(int index, E value) + { + checkSize(index); + isActive(); + + E oldValue = (E)values[index]; + values[index] = value; + return oldValue; + } + + @SuppressWarnings("unchecked") + @Override + protected Object clone() throws CloneNotSupportedException + { + try { + ResizeableArrayList v = (ResizeableArrayList) super.clone(); + v.values = Arrays.copyOf(values, values.length); + v.size = size; + return v; + } catch (CloneNotSupportedException e) { + throw new InternalError(); + } + } + + private void isActive() + { + if (!active) + { + throw new IllegalStateException("Not active"); + } + } + + void activate() + { + active = true; + } + + void deactivate() + { + active = false; + for (int i = 0 ; i < size ; i++) + { + values[i] = null; + } + } + + /** + * Sort the elements, in-place, contained in this list using {@link Comparator#} + * + * @param comparator The {@link Comparator} to sort with + */ + @SuppressWarnings("unchecked") + public void sort(Comparator comparator) + { + DualPivotQuickSort.sort((E[])values, 0, size -1 , comparator); + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index bdec26555..fda8d0b41 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -20,7 +20,6 @@ package org.alfresco.solr; import static java.util.Arrays.asList; import static java.util.Arrays.stream; -import static java.util.Optional.empty; import static java.util.Optional.ofNullable; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLID; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXCOMMITTIME; @@ -74,9 +73,6 @@ import static org.alfresco.solr.AlfrescoSolrDataModel.getAclChangeSetDocumentId; import static org.alfresco.solr.AlfrescoSolrDataModel.getAclDocumentId; import static org.alfresco.solr.utils.Utils.notNullOrEmpty; import static org.alfresco.util.ISO8601DateFormat.isTimeComponentDefined; -import static org.alfresco.service.cmr.security.AuthorityType.EVERYONE; -import static org.alfresco.service.cmr.security.AuthorityType.GROUP; -import static org.alfresco.service.cmr.security.AuthorityType.GUEST; import java.io.File; import java.io.IOException; @@ -825,7 +821,7 @@ public class SolrInformationServer implements InformationServer } @Override - public List getDocsWithUncleanContent() throws IOException + public List getDocsWithUncleanContent(int start, int rows) throws IOException { RefCounted refCounted = null; try @@ -850,7 +846,7 @@ public class SolrInformationServer implements InformationServer * in current snapshot of the index. * * The code below runs every two minutes and purges transactions from the - * cleanContentCache that is more than 20 minutes old. + * cleanContentCache that is more then 20 minutes old. * */ long purgeTime = System.currentTimeMillis(); @@ -884,9 +880,6 @@ public class SolrInformationServer implements InformationServer delegatingCollector.setLastDelegate(collector); searcher.search(documentsWithOutdatedContentQuery(), delegatingCollector); - LOGGER.debug("{}-[CORE {}] Processing {} documents with content to be indexed", Thread.currentThread().getId(), core.getName(), collector.getTotalHits()); - - if(collector.getTotalHits() == 0) { LOGGER.debug("No documents with outdated text content have been found."); @@ -1677,11 +1670,11 @@ public class SolrInformationServer implements InformationServer nmdp.setToNodeId(node.getId()); nmdp.setMaxResults(Integer.MAX_VALUE); - Optional> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); - if (nodeMetaDatas.isEmpty() || nodeMetaDatas.get().isEmpty()) return; + if (nodeMetaDatas.isEmpty()) return; - NodeMetaData nodeMetaData = nodeMetaDatas.get().iterator().next(); + NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next(); if (node.getTxnId() == Long.MAX_VALUE) { LOGGER.debug("Node {} index request is part of a re-index.", node.getId()); @@ -1820,8 +1813,8 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeTxnId(true); nmdp.setMaxResults(1); // Gets only one - Optional> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); - allNodeMetaDatas.addAll(nodeMetaDatas.orElse(Collections.emptyList())); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); + allNodeMetaDatas.addAll(nodeMetaDatas); } return allNodeMetaDatas; @@ -1961,14 +1954,7 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludePaths(false); nmdp.setIncludeParentAssociations(false); nmdp.setMaxResults(Integer.MAX_VALUE); - - Optional> nodesMetaDataFromRepository = getNodesMetaDataFromRepository(nmdp); - if (nodesMetaDataFromRepository.isEmpty()) - { - // Using exception for flow handling to jump to single node processing. - throw new Exception("Error loading node metadata from repository for bulk delete."); - } - nodeMetaDatas.addAll(nodesMetaDataFromRepository.get()); + nodeMetaDatas.addAll(getNodesMetaDataFromRepository(nmdp)); } for (NodeMetaData nodeMetaData : nodeMetaDatas) @@ -2002,15 +1988,10 @@ public class SolrInformationServer implements InformationServer // Fetches bulk metadata nmdp.setMaxResults(Integer.MAX_VALUE); - Optional> nodesMetaDataFromRepository = getNodesMetaDataFromRepository(nmdp); - if (nodesMetaDataFromRepository.isEmpty()) - { - // Using exception for flow handling to jump to single node processing. - throw new Exception("Error loading node metadata from repository for bulk delete."); - } + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); NEXT_NODE: - for (NodeMetaData nodeMetaData : nodesMetaDataFromRepository.get()) + for (NodeMetaData nodeMetaData : nodeMetaDatas) { long start = System.nanoTime(); @@ -2255,7 +2236,7 @@ public class SolrInformationServer implements InformationServer dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields() .stream() .filter(field -> field.getField().startsWith("text@sd___@")) - .forEach(field -> addStringProperty(valueHolder, field, value, locale)); + .forEach(field -> addStringProperty(valueHolder, field, value, locale, definition)); } else { dataModel.getIndexedFieldNamesForProperty(propertyQName).getFields() @@ -2264,7 +2245,7 @@ public class SolrInformationServer implements InformationServer { setUnitOfTimeFields(valueHolder, field.getField(), value.getValue(), definition.getDataType()); } - addStringProperty(valueHolder, field, value, locale); + addStringProperty(valueHolder, field, value, locale, definition); }); } } @@ -3266,10 +3247,17 @@ public class SolrInformationServer implements InformationServer */ private String addTenantToAuthority(String authority, String tenant) { - AuthorityType authorityType = AuthorityType.getAuthorityType(authority); - if ((authorityType == GROUP || authorityType == EVERYONE || authorityType == GUEST) && !tenant.isEmpty()) + switch (AuthorityType.getAuthorityType(authority)) { - return authority + "@" + tenant; + case GROUP: + case EVERYONE: + case GUEST: + if (tenant.length() != 0) + { + return authority + "@" + tenant; + } + default: + break; } return authority; } @@ -3436,11 +3424,11 @@ public class SolrInformationServer implements InformationServer nmdp.setIncludeTxnId(true); nmdp.setMaxResults(1); // Gets only one - Optional> nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); + Collection nodeMetaDatas = getNodesMetaDataFromRepository(nmdp); - if (nodeMetaDatas.isPresent() && !nodeMetaDatas.get().isEmpty()) + if (!nodeMetaDatas.isEmpty()) { - NodeMetaData nodeMetaData = nodeMetaDatas.get().iterator().next(); + NodeMetaData nodeMetaData = nodeMetaDatas.iterator().next(); // Only cascade update nods we know can not have changed and must be in this shard // Node in the current TX will be explicitly updated in the outer loop @@ -3749,7 +3737,7 @@ public class SolrInformationServer implements InformationServer return "\u0000" + locale.getLanguage() + "\u0000" + property.getValue(); } - private void addStringProperty(BiConsumer consumer, FieldInstance field, StringPropertyValue property, PropertyValue localeProperty) + private void addStringProperty(BiConsumer consumer, FieldInstance field, StringPropertyValue property, PropertyValue localeProperty, PropertyDefinition definition) { consumer.accept(field.getField(), field.isLocalised() ? getLocalisedValue(property, localeProperty) : property.getValue()); } @@ -3921,30 +3909,21 @@ public class SolrInformationServer implements InformationServer return nodeMetaData; } - /** - * Get the metadata for the specified nodes from the repository. - * - * @param parameters A parameters object containing either a list of nodes ({@link NodeMetaDataParameters#getNodeIds}) - * or a node range ({@link NodeMetaDataParameters#getFromNodeId} and {@link NodeMetaDataParameters#getToNodeId}). - * @return Either the metadata returned by the repository, or null if there was a problem. - */ - private Optional> getNodesMetaDataFromRepository(NodeMetaDataParameters parameters) + private Collection getNodesMetaDataFromRepository(NodeMetaDataParameters parameters) { - Collection nodeMetaDataCollection = null; try { - return Optional.of(notNullOrEmpty(repositoryClient.getNodesMetaData(parameters))); + return notNullOrEmpty(repositoryClient.getNodesMetaData(parameters)); } catch (JSONException exception) { - // The exception has been already logged in repositoryClient and could be huge. Simply log a reference to it here. - LOGGER.debug("JSON exception swallowed by SolrInformationServer."); - return empty(); + // Nothing to be done here: the exception has been already logged in repositoryClient + return Collections.emptyList(); } catch (Exception exception) { LOGGER.error("Unable to get nodes metadata from repository. See the stacktrace below for further details.", exception); - return empty(); + return Collections.emptyList(); } } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java index a16bde482..7d578e99c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/cache/CacheConstants.java @@ -32,9 +32,9 @@ public class CacheConstants // public static String ALFRESCO_READER_TO_ACL_IDS_CACHE = "alfrescoReaderToAclIdsCache"; // public static String ALFRESCO_DENY_TO_ACL_IDS_CACHE = "alfrescoDenyToAclIdsCache"; - public final static String ALFRESCO_AUTHORITY_CACHE = "alfrescoAuthorityCache"; - public final static String ALFRESCO_OWNERLOOKUP_CACHE = "alfrescoOwnerCache"; - public final static String ALFRESCO_READER_CACHE = "alfrescoReaderCache"; - public final static String ALFRESCO_DENIED_CACHE = "alfrescoDeniedCache"; - public final static String ALFRESCO_PATH_CACHE = "alfrescoPathCache"; + public static String ALFRESCO_AUTHORITY_CACHE = "alfrescoAuthorityCache"; + public static String ALFRESCO_OWNERLOOKUP_CACHE = "alfrescoOwnerCache"; + public static String ALFRESCO_READER_CACHE = "alfrescoReaderCache"; + public static String ALFRESCO_DENIED_CACHE = "alfrescoDeniedCache"; + public static String ALFRESCO_PATH_CACHE = "alfrescoPathCache"; } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java index 3d46c0d48..59eed0fe2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java @@ -5472,6 +5472,8 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants throw new RuntimeException("Error analyzing multiTerm term: " + part, e); } } + + private boolean analyzeRangeTerms = true; protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) { final BytesRef start; @@ -5480,13 +5482,13 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants if (part1 == null) { start = null; } else { - start = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part1) : new BytesRef(part1); + start = analyzeRangeTerms ? analyzeMultitermTerm(field, part1) : new BytesRef(part1); } if (part2 == null) { end = null; } else { - end = getAnalyzeRangeTerms() ? analyzeMultitermTerm(field, part2) : new BytesRef(part2); + end = analyzeRangeTerms ? analyzeMultitermTerm(field, part2) : new BytesRef(part2); } final TermRangeQuery query = new TermRangeQuery(field, start, end, startInclusive, endInclusive); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java index d59ff05bd..18affeca5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/SolrContainerScorer.java @@ -37,6 +37,10 @@ import org.apache.lucene.search.Weight; */ public class SolrContainerScorer extends Scorer { + // Unused + Weight weight; + + SolrContainerScorerDocIdSetIterator iterator; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java index 32adf6761..d0a19474f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java @@ -43,7 +43,8 @@ public abstract class AbstractTracker implements Tracker static final long TIME_STEP_32_DAYS_IN_MS = 1000 * 60 * 60 * 24 * 32L; static final long TIME_STEP_1_HR_IN_MS = 60 * 60 * 1000L; static final String SHARD_METHOD_DBID = "DB_ID"; - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTracker.class); + + protected final Logger logger = LoggerFactory.getLogger(getClass()); protected Properties props; protected SOLRAPIClient client; @@ -56,6 +57,9 @@ public abstract class AbstractTracker implements Tracker private int maxLiveSearchers; private volatile boolean shutdown = false; + private Semaphore runLock = new Semaphore(1, true); + private Semaphore writeLock = new Semaphore(1, true); + protected volatile TrackerState state; protected int shardCount; protected int shardInstance; @@ -70,6 +74,11 @@ public abstract class AbstractTracker implements Tracker protected final Type type; protected final String trackerId; + /* + * A thread handler can be used by subclasses, but they have to intentionally instantiate it. + */ + ThreadHandler threadHandler; + /** * Default constructor, strictly for testing. */ @@ -159,9 +168,9 @@ public abstract class AbstractTracker implements Tracker { String iterationId = "IT #" + System.currentTimeMillis(); - if(getRunLock().availablePermits() == 0) + if(runLock.availablePermits() == 0) { - LOGGER.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); + logger.info("[{} / {} / {}] Tracker already registered.", coreName, trackerId, iterationId); return; } @@ -171,7 +180,7 @@ public abstract class AbstractTracker implements Tracker * The runLock ensures that for each tracker type (metadata, content, commit, cascade) only one tracker will * be running at a time. */ - getRunLock().acquire(); + runLock.acquire(); if (state==null && Boolean.parseBoolean(System.getProperty("alfresco.test", "false"))) { @@ -182,7 +191,7 @@ public abstract class AbstractTracker implements Tracker { this.state = getTrackerState(); - LOGGER.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); + logger.debug("[{} / {} / {}] Global Tracker State set to: {}", coreName, trackerId, iterationId, this.state.toString()); this.state.setRunning(true); } else @@ -200,25 +209,28 @@ public abstract class AbstractTracker implements Tracker catch(IndexTrackingShutdownException t) { setRollback(true, t); - LOGGER.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); + logger.info("[{} / {} / {}] Tracking cycle stopped. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); } catch(Throwable t) { setRollback(true, t); if (t instanceof SocketTimeoutException || t instanceof ConnectException) { - LOGGER.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); - LOGGER.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); + logger.warn("[{} / {} / {}] Tracking communication timed out. See the stacktrace below for further details.", coreName, trackerId, iterationId); + if (logger.isDebugEnabled()) + { + logger.debug("[{} / {} / {}] Stack trace", coreName, trackerId, iterationId, t); + } } else { - LOGGER.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); + logger.error("[{} / {} / {}] Tracking failure. See the stacktrace below for further details.", coreName, trackerId, iterationId, t); } } } catch (InterruptedException e) { - LOGGER.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); + logger.error("[{} / {} / {}] Semaphore interruption. See the stacktrace below for further details.", coreName, trackerId, iterationId, e); } finally { @@ -228,7 +240,7 @@ public abstract class AbstractTracker implements Tracker state.setRunning(false); state.setCheck(false); }); - getRunLock().release(); + runLock.release(); } } @@ -272,6 +284,30 @@ public abstract class AbstractTracker implements Tracker } } + /** + * Allows time for the scheduled asynchronous tasks to complete + */ + synchronized void waitForAsynchronous() + { + AbstractWorkerRunnable currentRunnable = this.threadHandler.peekHeadReindexWorker(); + while (currentRunnable != null) + { + checkShutdown(); + synchronized (this) + { + try + { + wait(100); + } + catch (InterruptedException e) + { + // Nothing to be done here + } + } + currentRunnable = this.threadHandler.peekHeadReindexWorker(); + } + } + int getMaxLiveSearchers() { return maxLiveSearchers; @@ -301,19 +337,21 @@ public abstract class AbstractTracker implements Tracker public void shutdown() { setShutdown(true); + if(this.threadHandler != null) + { + threadHandler.shutDownThreadPool(); + } } - /** - * Trackers implementing this method should decide if the Write Lock is applied - * globally for every Tracker Thread (static) or locally for each running Thread - */ - public abstract Semaphore getWriteLock(); + public Semaphore getWriteLock() + { + return this.writeLock; + } - /** - * Trackers implementing this method should decide if the Run Lock is applied - * globally for every Tracker Thread (static) or locally for each running Thread - */ - public abstract Semaphore getRunLock(); + Semaphore getRunLock() + { + return this.runLock; + } public Properties getProps() { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java similarity index 54% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java index 40c126319..2aa335d98 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java @@ -1,56 +1,68 @@ -/* - * Copyright (C) 2005-2020 Alfresco Software Limited. - * This file is part of Alfresco - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Synchronous Tracking Worker. - * - * @author aborroy - * - */ -public abstract class AbstractWorker -{ - protected final static Logger LOGGER = LoggerFactory.getLogger(AbstractWorker.class); - - public void run() - { - boolean failed = true; - Exception failCausedBy = null; - try - { - doWork(); - failed = false; - } - catch (Exception e) - { - LOGGER.warn("Index tracking batch hit an unrecoverable error ", e); - failCausedBy = e; - } - finally - { - if (failed) - { - onFail(failCausedBy); - } - } - } - - abstract protected void doWork() throws Exception; - - abstract protected void onFail(Throwable failCausedBy); -} +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +abstract class AbstractWorkerRunnable implements Runnable +{ + protected final static Logger log = LoggerFactory.getLogger(AbstractWorkerRunnable.class); + + QueueHandler queueHandler; + + public AbstractWorkerRunnable(QueueHandler qh) + { + this.queueHandler = qh; + } + + /* + * (non-Javadoc) + * @see java.lang.Runnable#run() + */ + @Override + public void run() + { + boolean failed = true; + Exception failCausedBy = null; + try + { + doWork(); + failed = false; + } + catch (Exception e) + { + log.warn("Index tracking batch hit an unrecoverable error ", e); + failCausedBy = e; + } + finally + { + // Triple check that we get the queue state right + queueHandler.removeFromQueueAndProdHead(this); + if(failed) + { + onFail(failCausedBy); + } + } + } + + abstract protected void doWork() throws Exception; + + abstract protected void onFail(Throwable failCausedBy); +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index 17ef3081a..722e6ba26 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -20,22 +20,13 @@ package org.alfresco.solr.tracker; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import com.google.common.collect.Lists; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.index.shard.ShardMethodEnum; @@ -65,11 +56,8 @@ public class AclTracker extends AbstractTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class); - private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 2000; - private static final int DEFAULT_ACL_BATCH_SIZE = 100; - private static final int DEFAULT_ACL_TRACKER_MAX_PARALLELISM = 32; - - private int aclTrackerParallelism; + private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 100; + private static final int DEFAULT_ACL_BATCH_SIZE = 10; private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE; private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE; @@ -81,23 +69,6 @@ public class AclTracker extends AbstractTracker private ConcurrentLinkedQueue aclsToIndex = new ConcurrentLinkedQueue(); private ConcurrentLinkedQueue aclsToPurge = new ConcurrentLinkedQueue(); private DocRouter docRouter; - - private ForkJoinPool forkJoinPool; - - // Share run and write locks across all AclTracker threads - private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } - /** * Default constructor, for testing. @@ -111,18 +82,11 @@ public class AclTracker extends AbstractTracker String coreName, InformationServer informationServer) { super(p, client, coreName, informationServer, Tracker.Type.ACL); - changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", - String.valueOf(DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE))); - aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", String.valueOf(DEFAULT_ACL_BATCH_SIZE))); + changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", "100")); + aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", "10")); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); - - aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.aclTrackerMaxParallelism", - String.valueOf(DEFAULT_ACL_TRACKER_MAX_PARALLELISM))); - forkJoinPool = new ForkJoinPool(aclTrackerParallelism); - - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + threadHandler = new ThreadHandler(p, coreName, "AclTracker"); } @Override @@ -275,7 +239,7 @@ public class AclTracker extends AbstractTracker } } - protected void purgeAclChangeSets() throws IOException, JSONException + protected void purgeAclChangeSets() throws AuthenticationException, IOException, JSONException { while (aclChangeSetsToPurge.peek() != null) { @@ -290,7 +254,7 @@ public class AclTracker extends AbstractTracker } - protected void purgeAcls() throws IOException, JSONException + protected void purgeAcls() throws AuthenticationException, IOException, JSONException { while (aclsToPurge.peek() != null) { @@ -358,6 +322,10 @@ public class AclTracker extends AbstractTracker /** * Checks the first and last TX time + * @param state the state of this tracker + * @throws AuthenticationException + * @throws IOException + * @throws JSONException */ private void checkRepoAndIndexConsistency(TrackerState state) throws AuthenticationException, IOException, JSONException { @@ -374,13 +342,13 @@ public class AclTracker extends AbstractTracker AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0); long firstChangeSetCommitTime = firstChangeSet.getCommitTimeMs(); state.setLastGoodChangeSetCommitTimeInIndex(firstChangeSetCommitTime); - setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state); + setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets, state); } } if (!state.isCheckedFirstAclTransactionTime()) { - firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1); + firstChangeSets = client.getAclChangeSets(null, 0l, null, 2000L, 1); if (!firstChangeSets.getAclChangeSets().isEmpty()) { AclChangeSet firstAclChangeSet= firstChangeSets.getAclChangeSets().get(0); @@ -413,10 +381,10 @@ public class AclTracker extends AbstractTracker { if (firstChangeSets == null) { - firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1); + firstChangeSets = client.getAclChangeSets(null, 0l, null, 2000L, 1); } - setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state); + setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets, state); Long maxChangeSetCommitTimeInRepo = firstChangeSets.getMaxChangeSetCommitTime(); Long maxChangeSetIdInRepo = firstChangeSets.getMaxChangeSetId(); if (maxChangeSetCommitTimeInRepo != null && maxChangeSetIdInRepo != null) @@ -496,7 +464,7 @@ public class AclTracker extends AbstractTracker } else { - HashSet alreadyFound = new HashSet<>(changeSetsFound.getDeque()); + HashSet alreadyFound = new HashSet(changeSetsFound.getDeque()); for(AclChangeSet aclChangeSet : aclChangeSets.getAclChangeSets()) { if(!alreadyFound.contains(aclChangeSet)) @@ -531,7 +499,7 @@ public class AclTracker extends AbstractTracker } IOpenBitSet aclTxIdsInDb = infoSrv.getOpenBitSetInstance(); - long lastAclTxCommitTime = firstChangeSetCommitTimex; + Long lastAclTxCommitTime = Long.valueOf(firstChangeSetCommitTimex); if (fromTime != null) { lastAclTxCommitTime = fromTime; @@ -541,7 +509,7 @@ public class AclTracker extends AbstractTracker Long minAclTxId = null; long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention(); AclChangeSets aclTransactions; - BoundedDeque changeSetsFound = new BoundedDeque<>(100); + BoundedDeque changeSetsFound = new BoundedDeque(100); DO: do { aclTransactions = getSomeAclChangeSets(changeSetsFound, lastAclTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, @@ -551,14 +519,14 @@ public class AclTracker extends AbstractTracker // include if (toTime != null) { - if (set.getCommitTimeMs() > toTime) + if (set.getCommitTimeMs() > toTime.longValue()) { break DO; } } if (toAclTx != null) { - if (set.getId() > toAclTx) + if (set.getId() > toAclTx.longValue()) { break DO; } @@ -584,11 +552,44 @@ public class AclTracker extends AbstractTracker return this.infoSrv.reportAclTransactionsInIndex(minAclTxId, aclTxIdsInDb, maxAclTxId); } + + public List getFullNodesForDbTransaction(Long txid) + { + try + { + GetNodesParameters gnp = new GetNodesParameters(); + ArrayList txs = new ArrayList(); + txs.add(txid); + gnp.setTransactionIds(txs); + gnp.setStoreProtocol(storeRef.getProtocol()); + gnp.setStoreIdentifier(storeRef.getIdentifier()); + return client.getNodes(gnp, Integer.MAX_VALUE); + } + catch (IOException e) + { + throw new AlfrescoRuntimeException("Failed to get nodes", e); + } + catch (JSONException e) + { + throw new AlfrescoRuntimeException("Failed to get nodes", e); + } + catch (AuthenticationException e) + { + throw new AlfrescoRuntimeException("Failed to get nodes", e); + } + } + + + + /** + * @param acltxid Long + * @return List + **/ public List getAclsForDbAclTransaction(Long acltxid) { try { - ArrayList answer = new ArrayList<>(); + ArrayList answer = new ArrayList(); AclChangeSets changeSet = client.getAclChangeSets(null, acltxid, null, acltxid+1, 1); List acls = client.getAcls(changeSet.getAclChangeSets(), null, Integer.MAX_VALUE); for (Acl acl : acls) @@ -597,7 +598,15 @@ public class AclTracker extends AbstractTracker } return answer; } - catch (IOException | JSONException | AuthenticationException e) + catch (IOException e) + { + throw new AlfrescoRuntimeException("Failed to get acls", e); + } + catch (JSONException e) + { + throw new AlfrescoRuntimeException("Failed to get acls", e); + } + catch (AuthenticationException e) { throw new AlfrescoRuntimeException("Failed to get acls", e); } @@ -625,14 +634,9 @@ public class AclTracker extends AbstractTracker } /** - * Every ACL Change Set contains a list of ACLs to be indexed. - * This method gets ACL Change Sets from Alfresco Repository to be indexed. - * - * The indexing is performed in batches of ACL Change Sets and the Tracker Status - * is updated in batched of ACLs. - * - * Tracker Status contains the Commit Time from the latest ACL Change Set indexed, - * so new operations can be retrieved from Repository starting with that time. + * @throws AuthenticationException + * @throws IOException + * @throws JSONException */ protected void trackAclChangeSets() throws AuthenticationException, IOException, JSONException { @@ -642,10 +646,9 @@ public class AclTracker extends AbstractTracker boolean upToDate = false; AclChangeSets aclChangeSets; BoundedDeque changeSetsFound = new BoundedDeque(100); + HashSet changeSetsIndexed = new LinkedHashSet(); long totalAclCount = 0; - int aclCount; - - LOGGER.info("{}-[CORE {}] Tracking ACLs", Thread.currentThread().getId(), coreName); + int aclCount = 0; do { @@ -661,12 +664,12 @@ public class AclTracker extends AbstractTracker this.state = getTrackerState(); - Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, - state.getLastChangeSetCommitTimeOnServer() == 0 ? state.getLastGoodChangeSetCommitTimeInIndex() - : state.getLastChangeSetCommitTimeOnServer()); + Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex()); aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing()); + setLastChangeSetIdAndCommitTimeInTrackerState(aclChangeSets, state); + if (aclChangeSets.getAclChangeSets().size() > 0) { LOGGER.info("{}-[CORE {}] Found {} ACL change sets after lastTxCommitTime {}, ACL Change Sets from {} to {}", @@ -683,63 +686,90 @@ public class AclTracker extends AbstractTracker Thread.currentThread().getId(), coreName, fromCommitTime); } - // Ignore indexed ACL Change Sets - aclChangeSets = new AclChangeSets(aclChangeSets.getAclChangeSets().stream() - .filter(changeSet -> { - try - { - boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && - infoSrv.aclChangeSetInIndex(changeSet.getId(), true)); - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed", - Thread.currentThread().getId(), coreName, changeSet.getId()); - } - return !isInIndex; - } - catch (IOException e) - { - LOGGER.warn( - "{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index", - Thread.currentThread().getId(), coreName, changeSet.getId(), e); - return true; - } - }) - .collect(Collectors.toList())); - - - - - // Make sure we do not go ahead of where we started - we will check the holes here - // correctly next time - if (aclChangeSets.getAclChangeSets().stream().anyMatch(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing())) + ArrayList changeSetBatch = new ArrayList(); + for (int i = 0; i < aclChangeSets.getAclChangeSets().size(); i++) { - break; + + AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(i); + + boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && + infoSrv.aclChangeSetInIndex(changeSet.getId(), true)); + + if (isInIndex) + { + // Logging progress for large ACL Change Set tracking every 100 tracked ACLs + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Change Set Id was already indexed: {}", + Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId()); + } + changeSetsFound.add(changeSet); + } + else + { + + // Logging progress for ACL Change Set + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Current Change Set Id to be indexed: {}", + Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId()); + } + + // Make sure we do not go ahead of where we started - we will check the holes here + // correctly next time + if (changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()) { + upToDate = true; + break; + } + + changeSetBatch.add(changeSet); + if (getAclCount(changeSetBatch) > changeSetAclsBatchSize) { + aclCount += indexBatchOfChangeSets(changeSetBatch); + totalAclCount += aclCount; + + for (AclChangeSet scheduled : changeSetBatch) { + changeSetsFound.add(scheduled); + changeSetsIndexed.add(scheduled); + } + changeSetBatch.clear(); + } + } + + if (aclCount > batchCount) { + if (super.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { + indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state); + long endElapsed = System.nanoTime(); + trackerStats.addElapsedAclTime(aclCount, endElapsed - startElapsed); + startElapsed = endElapsed; + aclCount = 0; + } + } + checkShutdown(); } - final AtomicInteger counter = new AtomicInteger(); - Collection> changeSetBatches = aclChangeSets.getAclChangeSets().stream() - .collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize)).values(); - - - for (List changeSetBatch : changeSetBatches) - { - aclCount = indexBatchOfChangeSets(changeSetBatch); - for (AclChangeSet indexed : changeSetBatch) - { - changeSetsFound.add(indexed); + if (!changeSetBatch.isEmpty()) { + if (getAclCount(changeSetBatch) > 0) { + aclCount += indexBatchOfChangeSets(changeSetBatch); + totalAclCount += aclCount; } - // Update last committed transactions - setLastChangeSetIdAndCommitTimeInTrackerState(changeSetBatch, state); - indexAclChangeSetAfterWorker(changeSetBatch, state); + for (AclChangeSet scheduled : changeSetBatch) { + changeSetsFound.add(scheduled); + changeSetsIndexed.add(scheduled); + } + changeSetBatch.clear(); + } + + if(changeSetsIndexed.size() > 0) + { + indexAclChangeSetAfterAsynchronous(changeSetsIndexed, state); long endElapsed = System.nanoTime(); trackerStats.addElapsedAclTime(aclCount, endElapsed-startElapsed); startElapsed = endElapsed; + aclCount = 0; } - } - catch(InterruptedException | ExecutionException e) + catch(InterruptedException e) { throw new IOException(e); } @@ -751,39 +781,32 @@ public class AclTracker extends AbstractTracker } while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false)); - LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount); - + LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount); + } - private void setLastChangeSetIdAndCommitTimeInTrackerState(List aclChangeSets, TrackerState state) + private void setLastChangeSetIdAndCommitTimeInTrackerState(AclChangeSets aclChangeSets, TrackerState state) { - - if (!aclChangeSets.isEmpty()) + Long maxChangeSetCommitTime = aclChangeSets.getMaxChangeSetCommitTime(); + if(maxChangeSetCommitTime != null) { - Long maxChangeSetCommitTime = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs(); - if(maxChangeSetCommitTime != null) - { - state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime); - } - - Long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId(); - if(maxChangeSetId != null) - { - state.setLastChangeSetIdOnServer(maxChangeSetId); - } + state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime); + } + + Long maxChangeSetId = aclChangeSets.getMaxChangeSetId(); + if(maxChangeSetId != null) + { + state.setLastChangeSetIdOnServer(maxChangeSetId); } - } - /** - * Index ACL Change Set transaction after ACLs has been indexed by the worker - */ - private void indexAclChangeSetAfterWorker(Collection changeSetsIndexed, TrackerState state) + private void indexAclChangeSetAfterAsynchronous(HashSet changeSetsIndexed, TrackerState state) throws IOException { + waitForAsynchronous(); for (AclChangeSet set : changeSetsIndexed) { - infoSrv.indexAclTransaction(set, true); + super.infoSrv.indexAclTransaction(set, true); // Acl change sets are ordered by commit time and tie-broken by id if (set.getCommitTimeMs() > state.getLastIndexedChangeSetCommitTime() || set.getCommitTimeMs() == state.getLastIndexedChangeSetCommitTime() @@ -794,6 +817,8 @@ public class AclTracker extends AbstractTracker } trackerStats.addChangeSetAcls(set.getAclCount()); } + changeSetsIndexed.clear(); + //super.infoSrv.commit(); } private int getAclCount(List changeSetBatch) @@ -806,20 +831,19 @@ public class AclTracker extends AbstractTracker return count; } - /** - * Index ACLs from ACL Change Sets contained in changeSetBatch - * When total ACL indexed count is greater than the specified for a single execution - * (maxAclsPerExecution), no more ACL Change Sets are processed. - * - * @param changeSetBatch List of ACL Change Sets to be indexed - * @return List of ACL Change Set indexed and Count of ACL indexed - */ - private int indexBatchOfChangeSets(List changeSetBatch) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { - // Exclude ACL Change Set with no ACLs inside - List nonEmptyChangeSets = changeSetBatch.stream() - .filter(set -> set.getAclCount() > 0) - .collect(Collectors.toList()); + private int indexBatchOfChangeSets(List changeSetBatch) throws AuthenticationException, IOException, JSONException + { + int aclCount = 0; + ArrayList nonEmptyChangeSets = new ArrayList(changeSetBatch.size()); + for (AclChangeSet set : changeSetBatch) + { + if (set.getAclCount() > 0) + { + nonEmptyChangeSets.add(set); + } + } + ArrayList aclBatch = new ArrayList(); List acls = client.getAcls(nonEmptyChangeSets, null, Integer.MAX_VALUE); if (LOGGER.isDebugEnabled()) @@ -827,27 +851,40 @@ public class AclTracker extends AbstractTracker LOGGER.debug("{}-[CORE {}] Found {} Acls from Acl Change Sets: {}", Thread.currentThread().getId(), coreName, acls.size(), nonEmptyChangeSets); } - - List> aclBatches = Lists.partition(acls, aclBatchSize); - return forkJoinPool.submit(() -> - aclBatches.parallelStream().map(batch -> { - new AclIndexWorker(batch).run(); - return batch.size(); - }).reduce(0, Integer::sum) - ).get(); + for (Acl acl : acls) + { + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Adding ACL {} to scheduled indexing job", Thread.currentThread().getId(), + coreName, acl.toString()); + } + aclBatch.add(acl); + if (aclBatch.size() > aclBatchSize) + { + aclCount += aclBatch.size(); + AclIndexWorkerRunnable aiwr = new AclIndexWorkerRunnable(this.threadHandler, aclBatch); + this.threadHandler.scheduleTask(aiwr); + aclBatch = new ArrayList(); + } + } + if (aclBatch.size() > 0) + { + aclCount += aclBatch.size(); + AclIndexWorkerRunnable aiwr = new AclIndexWorkerRunnable(this.threadHandler, aclBatch); + this.threadHandler.scheduleTask(aiwr); + aclBatch = new ArrayList(); + } + return aclCount; } - - /** - * ACL Indexer - */ - class AclIndexWorker extends AbstractWorker + class AclIndexWorkerRunnable extends AbstractWorkerRunnable { List acls; - AclIndexWorker(List acls) + AclIndexWorkerRunnable(QueueHandler queueHandler, List acls) { + super(queueHandler); this.acls = acls; } @@ -870,7 +907,7 @@ public class AclTracker extends AbstractTracker private List filterAcls(List acls) { - ArrayList filteredList = new ArrayList<>(acls.size()); + ArrayList filteredList = new ArrayList(acls.size()); for(Acl acl : acls) { if(docRouter.routeAcl(shardCount, shardInstance, acl)) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 7eb0fa437..4946b2869 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -18,28 +18,20 @@ */ package org.alfresco.solr.tracker; -import static java.util.stream.Collectors.joining; - -import static org.alfresco.solr.utils.Utils.notNullOrEmpty; - import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Semaphore; -import com.google.common.collect.Lists; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.solr.InformationServer; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.client.Transaction; +import org.apache.commons.codec.EncoderException; import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,29 +46,8 @@ import static org.alfresco.solr.utils.Utils.notNullOrEmpty; public class CascadeTracker extends AbstractTracker implements Tracker { - protected final static Logger LOGGER = LoggerFactory.getLogger(CascadeTracker.class); + protected final static Logger log = LoggerFactory.getLogger(CascadeTracker.class); - private static final int DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM = 32; - private static final int DEFAULT_CASCADE_NODE_BATCH_SIZE = 10; - - - // Share run and write locks across all CascadeTracker threads - private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private int cascadeBatchSize; - private ForkJoinPool forkJoinPool; - private int cascadeTrackerParallelism; - - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } public CascadeTracker(Properties p, SOLRAPIClient client, String coreName, @@ -84,15 +55,7 @@ public class CascadeTracker extends AbstractTracker implements Tracker { super(p, client, coreName, informationServer, Tracker.Type.CASCADE); - cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascadeTrackerMaxParallelism", - String.valueOf(DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM))); - - cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascadeNodeBatchSize", - String.valueOf(DEFAULT_CASCADE_NODE_BATCH_SIZE)));; - - forkJoinPool = new ForkJoinPool(cascadeTrackerParallelism); - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + threadHandler = new ThreadHandler(p, coreName, "CascadeTracker"); } CascadeTracker() @@ -101,7 +64,7 @@ public class CascadeTracker extends AbstractTracker implements Tracker } @Override - protected void doTrack(String iterationId) throws IOException, JSONException + protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException { // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); @@ -111,36 +74,38 @@ public class CascadeTracker extends AbstractTracker implements Tracker } } - public void maintenance() - { + public void maintenance() throws Exception { + } public boolean hasMaintenance() { return false; } - private void trackRepository(String iterationId) throws IOException, JSONException + private void trackRepository(String iterationId) throws IOException, AuthenticationException, JSONException, EncoderException { checkShutdown(); processCascades(iterationId); } - private void updateTransactionsAfterWorker(List txsIndexed) + private void updateTransactionsAfterAsynchronous(List txsIndexed) throws IOException { + waitForAsynchronous(); for (Transaction tx : txsIndexed) { super.infoSrv.updateTransaction(tx); } } - class CascadeIndexWorker extends AbstractWorker + class CascadeIndexWorkerRunnable extends AbstractWorkerRunnable { InformationServer infoServer; List nodes; - CascadeIndexWorker(List nodes, InformationServer infoServer) + CascadeIndexWorkerRunnable(QueueHandler queueHandler, List nodes, InformationServer infoServer) { + super(queueHandler); this.infoServer = infoServer; this.nodes = nodes; } @@ -154,12 +119,11 @@ public class CascadeTracker extends AbstractTracker implements Tracker @Override protected void onFail(Throwable failCausedBy) { - setRollback(true, failCausedBy); + setRollback(true, failCausedBy); } } - public void invalidateState() - { + public void invalidateState() { super.invalidateState(); infoSrv.setCleanCascadeTxnFloor(-1); } @@ -168,68 +132,52 @@ public class CascadeTracker extends AbstractTracker implements Tracker { int num = 50; List txBatch = null; - long totalUpdatedDocs = 0; - do { try { getWriteLock().acquire(); txBatch = infoSrv.getCascades(num); - - if (txBatch.size() > 0) - { - LOGGER.info("{}-[CORE {}] Found {} transactions, transactions from {} to {}", - Thread.currentThread().getId(), - coreName, - txBatch.size(), - txBatch.get(0), - txBatch.get(txBatch.size() - 1)); - } - else - { - LOGGER.info("{}-[CORE {}] No transaction found", - Thread.currentThread().getId(), coreName); - } - if(txBatch.size() == 0) { + //No transactions to process for cascades. return; } - ArrayList txIds = new ArrayList<>(); - Set txIdSet = new HashSet<>(); + ArrayList txIds = new ArrayList(); + Set txIdSet = new HashSet(); for (Transaction tx : txBatch) { txIds.add(tx.getId()); txIdSet.add(tx.getId()); } List nodeMetaDatas = infoSrv.getCascadeNodes(txIds); - Integer processedCascades = 0; if(nodeMetaDatas.size() > 0) { - List> nodeBatches = Lists.partition(nodeMetaDatas, cascadeBatchSize); + LinkedList stack = new LinkedList(); + stack.addAll(nodeMetaDatas); + int batchSize = 10; - processedCascades = forkJoinPool.submit( () -> - nodeBatches.parallelStream().map( batch -> { - - CascadeIndexWorker worker = new CascadeIndexWorker(batch, infoSrv); - worker.run(); - - if (LOGGER.isTraceEnabled()) - { - String nodes = notNullOrEmpty(batch).stream() - .map(NodeMetaData::getId) - .map(Object::toString) - .collect(joining(",")); - LOGGER.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); - } - return batch.size(); - }).reduce(0, Integer::sum) - ).get(); + do { + List batch = new ArrayList(); + while (batch.size() < batchSize && stack.size() > 0) { + batch.add(stack.removeFirst()); + } + CascadeIndexWorkerRunnable worker = new CascadeIndexWorkerRunnable(this.threadHandler, batch, infoSrv); + + if (logger.isTraceEnabled()) + { + String nodes = notNullOrEmpty(batch).stream() + .map(NodeMetaData::getId) + .map(Object::toString) + .collect(joining(",")); + logger.trace("[{} / {} / {} / {}] Worker has been created for nodes {}", coreName, trackerId, iterationId, worker.hashCode(), nodes); + } + this.threadHandler.scheduleTask(worker); + } + while (stack.size() > 0); } //Update the transaction records. - updateTransactionsAfterWorker(txBatch); - totalUpdatedDocs += processedCascades; + updateTransactionsAfterAsynchronous(txBatch); } catch (AuthenticationException e) { @@ -243,17 +191,11 @@ public class CascadeTracker extends AbstractTracker implements Tracker { throw new IOException(e); } - catch (ExecutionException e) - { - e.printStackTrace(); - } finally { getWriteLock().release(); } + } while(txBatch.size() > 0); - - LOGGER.info("{}-[CORE {}] Updated {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs); - } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java index 3086d146d..0c29a46b4 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java @@ -23,11 +23,8 @@ import static java.util.Optional.empty; import static java.util.Optional.ofNullable; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import org.alfresco.solr.InformationServer; @@ -52,21 +49,7 @@ public class CommitTracker extends AbstractTracker private Optional cascadeTracker = empty(); private AtomicInteger rollbackCount = new AtomicInteger(0); - protected final static Logger LOGGER = LoggerFactory.getLogger(CommitTracker.class); - - // Share run and write locks across all CommitTracker threads - private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } + protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class); /** * Default constructor, for testing. @@ -100,9 +83,6 @@ public class CommitTracker extends AbstractTracker commitInterval = Long.parseLong(p.getProperty("alfresco.commitInterval", "60000")); // Default: commit once per minute newSearcherInterval = Integer.parseInt(p.getProperty("alfresco.newSearcherInterval", "120000")); // Default: Open searchers every two minutes lastSearcherOpened = lastCommit = System.currentTimeMillis(); - - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } public boolean hasMaintenance() throws Exception @@ -214,17 +194,17 @@ public class CommitTracker extends AbstractTracker // Log reasons why the rollback is performed if (aclTracker.getRollbackCausedBy() != null) { - LOGGER.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy()); + log.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy()); } if (metadataTracker.getRollbackCausedBy() != null) { - LOGGER.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy()); + log.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy()); } } catch (Exception e) { - LOGGER.error("Rollback failed", e); + log.error("Rollback failed", e); } finally { diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java index b605c534a..b43854fda 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java @@ -16,22 +16,15 @@ * You should have received a copy of the GNU Lesser General Public License * along with Alfresco. If not, see . */ -package org.alfresco.solr.tracker; - -import com.google.common.collect.Lists; -import org.alfresco.solr.AlfrescoSolrDataModel.TenantDbId; -import org.alfresco.solr.InformationServer; -import org.alfresco.solr.client.SOLRAPIClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + package org.alfresco.solr.tracker; import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Semaphore; + +import org.alfresco.solr.AlfrescoSolrDataModel.TenantDbId; +import org.alfresco.solr.InformationServer; +import org.alfresco.solr.client.SOLRAPIClient; import static org.alfresco.solr.utils.Utils.notNullOrEmpty; @@ -43,45 +36,17 @@ import static org.alfresco.solr.utils.Utils.notNullOrEmpty; */ public class ContentTracker extends AbstractTracker implements Tracker { - protected final static Logger LOGGER = LoggerFactory.getLogger(ContentTracker.class); - - private static final int DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM = 32; - - private int contentTrackerParallelism; + private int contentReadBatchSize; private int contentUpdateBatchSize; - // Share run and write locks across all ContentTracker threads - private static final Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static final Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private ForkJoinPool forkJoinPool; - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } - - public ContentTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) + public ContentTracker(Properties p, SOLRAPIClient client, String coreName, + InformationServer informationServer) { super(p, client, coreName, informationServer, Tracker.Type.CONTENT); - int DEFAULT_CONTENT_UPDATE_BATCH_SIZE = 2000; - - contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", - String.valueOf(DEFAULT_CONTENT_UPDATE_BATCH_SIZE))); - - contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.contentTrackerMaxParallelism", - String.valueOf(DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM))); - - forkJoinPool = new ForkJoinPool(contentTrackerParallelism); - - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + contentReadBatchSize = Integer.parseInt(p.getProperty("alfresco.contentReadBatchSize", "100")); + contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", "1000")); + threadHandler = new ThreadHandler(p, coreName, "ContentTracker"); } ContentTracker() @@ -97,39 +62,51 @@ public class ContentTracker extends AbstractTracker implements Tracker long startElapsed = System.nanoTime(); checkShutdown(); + final int ROWS = contentReadBatchSize; + int start = 0; long totalDocs = 0L; checkShutdown(); while (true) { try { + getWriteLock().acquire(); - List docs = notNullOrEmpty(this.infoSrv.getDocsWithUncleanContent()); + List docs = notNullOrEmpty(infoSrv.getDocsWithUncleanContent(start, ROWS)); if (docs.isEmpty()) { - LOGGER.trace("No unclean document has been detected in the current ContentTracker cycle."); + logger.debug("No unclean document has been detected in the current ContentTracker cycle."); break; } - List> docBatches = Lists.partition(docs, contentUpdateBatchSize); - for (List batch : docBatches) + int docsUpdatedSinceLastCommit = 0; + for (TenantDbId doc : docs) { - Integer processedDocuments = forkJoinPool.submit(() -> - // Parallel task here, for example - batch.parallelStream().map(doc -> { - ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(doc, infoSrv); - ciwr.run(); - return 1; - }).reduce(0, Integer::sum) - ).get(); + ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(super.threadHandler, doc, infoSrv); + super.threadHandler.scheduleTask(ciwr); + docsUpdatedSinceLastCommit++; - long endElapsed = System.nanoTime(); - trackerStats.addElapsedContentTime(processedDocuments, endElapsed - startElapsed); - startElapsed = endElapsed; + if (docsUpdatedSinceLastCommit >= contentUpdateBatchSize) + { + super.waitForAsynchronous(); + checkShutdown(); + long endElapsed = System.nanoTime(); + trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed); + startElapsed = endElapsed; + docsUpdatedSinceLastCommit = 0; + } } + if (docsUpdatedSinceLastCommit > 0) + { + super.waitForAsynchronous(); + checkShutdown(); + //this.infoSrv.commit(); + long endElapsed = System.nanoTime(); + trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed); + } totalDocs += docs.size(); checkShutdown(); } @@ -139,8 +116,7 @@ public class ContentTracker extends AbstractTracker implements Tracker } } - LOGGER.info("{}-[CORE {}] Total number of docs with content updated: {} ", Thread.currentThread().getId(), coreName, totalDocs); - + logger.info("Total number of docs with content updated: {}", totalDocs); } catch(Exception e) { @@ -164,14 +140,16 @@ public class ContentTracker extends AbstractTracker implements Tracker this.infoSrv.setCleanContentTxnFloor(-1); } - class ContentIndexWorkerRunnable extends AbstractWorker + class ContentIndexWorkerRunnable extends AbstractWorkerRunnable { InformationServer infoServer; TenantDbId docRef; - ContentIndexWorkerRunnable(TenantDbId doc, InformationServer infoServer) + ContentIndexWorkerRunnable(QueueHandler queueHandler, TenantDbId docRef, InformationServer infoServer) { - this.docRef = doc; + super(queueHandler); + + this.docRef = docRef; this.infoServer = infoServer; } @@ -187,7 +165,7 @@ public class ContentTracker extends AbstractTracker implements Tracker protected void onFail(Throwable failCausedBy) { // This will be redone in future tracking operations - LOGGER.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy); + log.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy); } } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java index 45352c1e9..aec77956c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java @@ -46,8 +46,6 @@ import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.Optional; @@ -65,7 +63,6 @@ import java.util.Properties; */ public abstract class CoreStatePublisher extends AbstractTracker { - private static final Logger LOGGER = LoggerFactory.getLogger(CoreStatePublisher.class); DocRouter docRouter; private final boolean isMaster; @@ -121,7 +118,7 @@ public abstract class CoreStatePublisher extends AbstractTracker updateShardProperty(); if (shardProperty.isEmpty()) { - LOGGER.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); + logger.warn("Sharding property {} was set to {}, but no such property was found.", SHARD_KEY_KEY, shardKeyName); } }); } @@ -137,11 +134,11 @@ public abstract class CoreStatePublisher extends AbstractTracker { if (updatedShardProperty.isEmpty()) { - LOGGER.warn("The model defining {} property has been disabled", shardKeyName); + logger.warn("The model defining {} property has been disabled", shardKeyName); } else { - LOGGER.info("New {} property found for {}", SHARD_KEY_KEY, shardKeyName); + logger.info("New {} property found for {} ", SHARD_KEY_KEY, shardKeyName); } } shardProperty = updatedShardProperty; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java index 21d3b38ac..672313371 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterFactory.java @@ -18,7 +18,6 @@ */ package org.alfresco.solr.tracker; -import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,15 +44,14 @@ public class DocRouterFactory LOGGER.info("Sharding via DB_ID"); return new DBIDRouter(); case DB_ID_RANGE: - if(!properties.containsKey(SHARD_RANGE_KEY)) + if(properties.containsKey(SHARD_RANGE_KEY)) { - throw new AlfrescoRuntimeException("DB_ID_RANGE sharding requires the " + SHARD_RANGE_KEY + " property to be set."); + LOGGER.info("Sharding via DB_ID_RANGE"); + String[] pair =properties.getProperty(SHARD_RANGE_KEY).split("-"); + long start = Long.parseLong(pair[0]); + long end = Long.parseLong(pair[1]); + return new DBIDRangeRouter(start, end); } - LOGGER.info("Sharding via DB_ID_RANGE"); - String[] pair = properties.getProperty(SHARD_RANGE_KEY).split("-"); - long start = Long.parseLong(pair[0]); - long end = Long.parseLong(pair[1]); - return new DBIDRangeRouter(start, end); case ACL_ID: LOGGER.info("Sharding via ACL_ID"); return new ACLIDMurmurRouter(); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index c8443fe58..e69e6bdaf 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -18,7 +18,17 @@ */ package org.alfresco.solr.tracker; -import com.google.common.collect.Lists; +import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ConcurrentLinkedQueue; + import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.index.shard.ShardState; @@ -39,41 +49,17 @@ import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Semaphore; -import java.util.stream.Collectors; - -import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE; - /* * This tracks two things: transactions and metadata nodes * @author Ahmed Owian */ public class MetadataTracker extends CoreStatePublisher implements Tracker { - protected final static Logger LOGGER = LoggerFactory.getLogger(MetadataTracker.class); - - private static final int DEFAULT_METADATA_TRACKER_MAX_PARALLELISM = 32; - private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 2000; - private static final int DEFAULT_MAX_NUMBER_OF_TRANSACTIONS = 2000; - private static final int DEFAULT_NODE_BATCH_SIZE = 50; - private static final String DEFAULT_INITIAL_TRANSACTION_RANGE = "0-2000"; - - private int matadataTrackerParallelism; - private int transactionDocsBatchSize; - private int nodeBatchSize; - private int maxNumberOfTransactions; - + protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class); + private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100; + private static final int DEFAULT_NODE_BATCH_SIZE = 10; + private int transactionDocsBatchSize = DEFAULT_TRANSACTION_DOCS_BATCH_SIZE; + private int nodeBatchSize = DEFAULT_NODE_BATCH_SIZE; private ConcurrentLinkedQueue transactionsToReindex = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue transactionsToIndex = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue transactionsToPurge = new ConcurrentLinkedQueue<>(); @@ -84,22 +70,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker private final boolean isRunningInProduction = !Boolean.parseBoolean(System.getProperty("alfresco.test", "false")); - private ForkJoinPool forkJoinPool; - - // Share run and write locks across all MetadataTracker threads - private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } - /** * Check if nextTxCommitTimeService is available in the repository. * This service is used to find the next available transaction commit time from a given time, @@ -147,22 +117,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker InformationServer informationServer, boolean checkRepoServicesAvailability) { super(isMaster, p, client, coreName, informationServer, Tracker.Type.METADATA); - - transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", - String.valueOf(DEFAULT_TRANSACTION_DOCS_BATCH_SIZE))); - nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", - String.valueOf(DEFAULT_NODE_BATCH_SIZE))); - maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.maxNumberOfTransactions", String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS))); - matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadataTrackerMaxParallelism", - String.valueOf(DEFAULT_METADATA_TRACKER_MAX_PARALLELISM))); - - String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE).split("-"); + transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); + nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); + threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled(); - minTxnIdRange = new Pair<>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1])); - forkJoinPool = new ForkJoinPool(matadataTrackerParallelism); - - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); + String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", "0-2000").split("-"); + minTxnIdRange = new Pair(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1])); // In order to apply performance optimizations, checking the availability of Repo Web Scripts is required. // As these services are available from ACS 6.2 @@ -176,11 +136,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - LOGGER.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); } catch (Exception e) { - LOGGER.error("Checking nextTxCommitTimeService failed.", e); + log.error("Checking nextTxCommitTimeService failed.", e); } // Try invoking txIntervalCommitTime service @@ -193,12 +153,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " + + log.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " + "to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage()); } catch (Exception e) { - LOGGER.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); + log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e); } } } @@ -213,6 +173,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker @Override protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException { + log.debug("### MetadataTracker doTrack ###"); // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); if (modelTracker != null && modelTracker.hasModels()) @@ -249,11 +210,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException { + log.debug("####### MetadataTracker trackRepository Start #######"); checkShutdown(); - // Check we are tracking the correct repository TrackerState state = super.getTrackerState(); + log.debug("####### MetadataTracker check CYCLE #######"); + log.debug(String.format("%s ### state: %s", coreName, state.toString())); if(state.getTrackerCycles() == 0) { //We have a new tracker state so do the checks. @@ -293,7 +256,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { state.setCheckedLastTransactionTime(true); state.setCheckedFirstTransactionTime(true); - LOGGER.info("No transactions found - no verification required"); + log.info("No transactions found - no verification required"); firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); if (!firstTransactions.getTransactions().isEmpty()) @@ -301,7 +264,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Transaction firstTransaction = firstTransactions.getTransactions().get(0); long firstTransactionCommitTime = firstTransaction.getCommitTimeMs(); state.setLastGoodTxCommitTimeInIndex(firstTransactionCommitTime); - setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions); + setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); } } @@ -325,7 +288,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " + "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage()); } } @@ -344,20 +307,20 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if (setSize == 0) { - LOGGER.error("First transaction was not found with the correct timestamp."); - LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); - LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); + log.error("First transaction was not found with the correct timestamp."); + log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); + log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + log.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp"); } else if (setSize == 1) { state.setCheckedFirstTransactionTime(true); - LOGGER.info("Verified first transaction and timestamp in index"); + log.info("Verified first transaction and timestamp in index"); } else { - LOGGER.warn("Duplicate initial transaction found with correct timestamp"); + log.warn("Duplicate initial transaction found with correct timestamp"); } } } @@ -371,7 +334,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); } - setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions); + setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); Long maxTxnCommitTimeInRepo = firstTransactions.getMaxTxnCommitTime(); Long maxTxnIdInRepo = firstTransactions.getMaxTxnId(); if (maxTxnCommitTimeInRepo != null && maxTxnIdInRepo != null) @@ -379,19 +342,19 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Transaction maxTxInIndex = this.infoSrv.getMaxTransactionIdAndCommitTimeInIndex(); if (maxTxInIndex.getCommitTimeMs() > maxTxnCommitTimeInRepo) { - LOGGER.error("Last transaction was found in index with timestamp later than that of repository."); - LOGGER.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo); - LOGGER.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + log.error("Last transaction was found in index with timestamp later than that of repository."); + log.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo); + log.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + maxTxnCommitTimeInRepo); - LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); - LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); + log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); + log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + log.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp"); } else { state.setCheckedLastTransactionTime(true); - LOGGER.info("Verified last transaction timestamp in index less than or equal to that of repository."); + log.info("Verified last transaction timestamp in index less than or equal to that of repository."); } } } @@ -429,9 +392,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker for (Node node : nodes) { docCount++; - if (LOGGER.isDebugEnabled()) + if (log.isDebugEnabled()) { - LOGGER.debug(node.toString()); + log.debug(node.toString()); } this.infoSrv.indexNode(node, false); checkShutdown(); @@ -439,15 +402,14 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // Index the transaction doc after the node - if this is not found then a reindex will be done. this.infoSrv.indexTransaction(info, false); - LOGGER.info("INDEX ACTION - Transaction {} has been indexed", transactionId); + log.info("INDEX ACTION - Transaction {} has been indexed", transactionId); requiresCommit = true; trackerStats.addTxDocs(nodes.size()); - } else { - LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); + log.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); } } @@ -456,6 +418,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if(this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { checkShutdown(); + //this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed-startElapsed); startElapsed = endElapsed; @@ -487,7 +450,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker node.setTxnId(Long.MAX_VALUE); this.infoSrv.indexNode(node, false); - LOGGER.info("INDEX ACTION - Node {} has been reindexed", node.getId()); + log.info("INDEX ACTION - Node {} has been reindexed", node.getId()); requiresCommit = true; } checkShutdown(); @@ -528,9 +491,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker for (Node node : nodes) { docCount++; - if (LOGGER.isDebugEnabled()) + if (log.isDebugEnabled()) { - LOGGER.debug(node.toString()); + log.debug(node.toString()); } this.infoSrv.indexNode(node, true); checkShutdown(); @@ -538,11 +501,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // Index the transaction doc after the node - if this is not found then a reindex will be done. this.infoSrv.indexTransaction(info, true); - LOGGER.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId); + log.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId); } else { - LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); + log.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); } } @@ -585,7 +548,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker node.setTxnId(Long.MAX_VALUE); this.infoSrv.indexNode(node, true); - LOGGER.info("REINDEX ACTION - Node {} has been reindexed", node.getId()); + log.info("REINDEX ACTION - Node {} has been reindexed", node.getId()); requiresCommit = true; } checkShutdown(); @@ -607,7 +570,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if (query != null) { this.infoSrv.reindexNodeByQuery(query); - LOGGER.info("REINDEX ACTION - Nodes from query {} have been reindexed", query); + log.info("REINDEX ACTION - Nodes from query {} have been reindexed", query); requiresCommit = true; } checkShutdown(); @@ -632,7 +595,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByTransactionId(transactionId); requiresCommit = true; - LOGGER.info("PURGE ACTION - Purged transactionId {}", transactionId); + log.info("PURGE ACTION - Purged transactionId {}", transactionId); } checkShutdown(); } @@ -640,6 +603,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if(requiresCommit) { checkShutdown(); + //this.infoSrv.commit(); } } @@ -652,30 +616,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByNodeId(nodeId); - LOGGER.info("PURGE ACTION - Purged nodeId {}", nodeId); + log.info("PURGE ACTION - Purged nodeId {}", nodeId); } checkShutdown(); } } - /** - * The fromCommitTime tells getSomeTransactions() where to start, this actually fairly straight forward. - * - * What makes this code so tricky to understand is the state.getTimeToStopIndexing(). - * - * There are two scenarios to keep in mind: - * - * 1) Full re-index: In this scenario the state.getTimeToStopIndexing() will never stop the indexing. - * - * 2) Up-to-date indexing: This is where state.getTimeToStopIndexing() gets interesting. In this scenario - * the Solr index is already up to date with the repo and it is tracking new transactions. The state.getTimeToStopIndexing() - * in this scenario causes the getSomeTransactions() call to stop returning results if it finds a transaction - * beyond a specific point in time. This will break out of this loop and end the tracker run. - * - * The next time the metadata tracker runs the "continueState()" method applies the "hole retention" - * to state.getLastGoodTxCommitTimeInIndex(). This causes the state.getLastGoodTxCommitTimeInIndex() to scan - * for prior transactions that might have been missed. - */ protected Long getTxFromCommitTime(BoundedDeque txnsFound, long lastGoodTxCommitTimeInIndex) { if (txnsFound.size() > 0) { @@ -718,6 +664,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // step forward in time until we find something or hit the time bound // max id unbounded Long startTime = fromCommitTime == null ? 0L : fromCommitTime; + log.debug(String.format("#### %s MetadataTracker getSomeTransactions start time: %d end: %d", + this.coreName, + startTime, + endTime)); if(startTime == 0) { return client.getTransactions(startTime, @@ -727,7 +677,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker maxResults, shardstate); } - do { transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate); @@ -739,8 +688,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Long nextTxCommitTime = client.getNextTxCommitTime(coreName, startTime); if (nextTxCommitTime != -1) { - LOGGER.info("{}-[CORE {}] Advancing transactions from {} to {}", - Thread.currentThread().getId(), coreName, startTime, nextTxCommitTime); + log.info("Advancing transactions from {} to {}", startTime, nextTxCommitTime); transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate); } } @@ -752,154 +700,26 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker return transactions; } - /** - * When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction - * for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime. - * When there isn't nodes in that range, -1 is returned as commit times - * - * @param fromCommitTime Starting commit time to get transactions from Repository - * @param txnsFound List of transactions previously found - * @return List of transactions to be indexed - */ - private Transactions getDBIDRangeTransactions(Long fromCommitTime, BoundedDeque txnsFound) - throws NoSuchMethodException, AuthenticationException, IOException, JSONException, EncoderException - { - boolean shardOutOfRange = false; - - DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter; - Pair commitTimes = client.getTxIntervalCommitTime(coreName, - dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); - Long shardMinCommitTime = commitTimes.getFirst(); - Long shardMaxCommitTime = commitTimes.getSecond(); - - // Node Range it's not still available in repository - if (shardMinCommitTime == -1) - { - LOGGER.debug( - "{}-[CORE {}] [DB_ID_RANGE] No nodes in range [{}-{}] " - + "exist in the repository. Indexing only latest transaction.", - Thread.currentThread().getId(), coreName, dbIdRangeRouter.getStartRange(), - dbIdRangeRouter.getEndRange()); - shardOutOfRange = true; - } - if (fromCommitTime > shardMaxCommitTime) - { - LOGGER.debug( - "{}-[CORE {}] [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. " - + "Indexing only latest transaction.", - Thread.currentThread().getId(), coreName, dbIdRangeRouter.getStartRange(), - dbIdRangeRouter.getEndRange()); - shardOutOfRange = true; - } - // Initial commit time for Node Range is greater than calculated from commit time - if (fromCommitTime < shardMinCommitTime) - { - LOGGER.debug("{}-[CORE {}] [DB_ID_RANGE] Skipping transactions from {} to {}", - Thread.currentThread().getId(), coreName, fromCommitTime, shardMinCommitTime); - fromCommitTime = shardMinCommitTime; - } - - Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, - state.getTimeToStopIndexing()); - - - // When transactions are out of Shard range, only the latest transaction needs to be indexed - // in order to preserve the state up-to-date of the MetadataTracker - if (shardOutOfRange) - { - Transaction latestTransaction = new Transaction(); - latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); - latestTransaction.setId(transactions.getMaxTxnId()); - transactions = new Transactions( - Arrays.asList(latestTransaction), - transactions.getMaxTxnCommitTime(), - transactions.getMaxTxnId()); - } - - return transactions; - } - - /** - * Remove transactions already present in SOLR index - * - * @param transactions List of transactions to be indexed - * @return List of transactions not indexed in SOLR index - */ - private Transactions removeIndexedTransactions(Transactions transactions) - { - return new Transactions(transactions.getTransactions().stream() - .filter(transaction -> { - try - { - boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() && - infoSrv.txnInIndex(transaction.getId(), true)); - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed", - Thread.currentThread().getId(), coreName, transaction.getId()); - } - return !isInIndex; - } - catch (IOException e) - { - LOGGER.warn( - "{}-[CORE {}] Error catched while checking if Transaction Id {} was in index", - Thread.currentThread().getId(), coreName, transaction.getId(), e); - return true; - } - }) - .collect(Collectors.toList())); - - } - - - //fixme remove - /** - * Keep only transactions previous to node transaction Id - * - * @param transactions List of transactions from Repository - * @param node Last Node indexed in the cycle - * @return Filtered list of transactions - */ - private Transactions filterTransactionsByNode(Transactions transactions, Node node) - { - return new Transactions(transactions.getTransactions().stream() - .filter(transaction -> { - return transaction.getId() < node.getTxnId(); - }) - .collect(Collectors.toList())); - } - - /** - * Indexing new transactions from repository in batches of "transactionDocsBatchSize" size. - * - * Additionally, the nodes inside a transaction batch are indexed in batches of "nodeBatchSize" size. - * - * @throws AuthenticationException - * @throws IOException - * @throws JSONException - * @throws EncoderException - */ protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException { long startElapsed = System.nanoTime(); - + boolean upToDate = false; Transactions transactions; BoundedDeque txnsFound = new BoundedDeque(100); - int totalUpdatedDocs = 0; - - LOGGER.info("{}-[CORE {}] Starting metadata tracker execution", Thread.currentThread().getId(), coreName); - + HashSet txsIndexed = new LinkedHashSet<>(); + long totalUpdatedDocs = 0; + int docCount = 0; + do { - try { /* * This write lock is used to lock out the Commit Tracker. The ensures that the MetaDataTracker will * not be indexing content while commits or rollbacks are occurring. */ + getWriteLock().acquire(); /* @@ -907,94 +727,184 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * tracker state could have been invalidated due to a rollback by the CommitTracker. * In this case the state will revert to the last transaction state record in the index. */ + this.state = getTrackerState(); - Long fromCommitTime = getTxFromCommitTime(txnsFound, - state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime()); - // Get transaction list to be indexed + /* + * The fromCommitTime tells getSomeTransactions() where to start, this actually fairly straight forward. + * + * What makes this code so tricky to understand is the state.getTimeToStopIndexing(). + * + * There are two scenarios to keep in mind: + * + * 1) Full re-index: In this scenario the state.getTimeToStopIndexing() will never stop the indexing. + * + * 2) Up-to-date indexing: This is where state.getTimeToStopIndexing() gets interesting. In this scenario + * the Solr index is already up to date with the repo and it is tracking new transactions. The state.getTimeToStopIndexing() + * in this scenario causes the getSomeTransactions() call to stop returning results if it finds a transaction + * beyond a specific point in time. This will break out of this loop and end the tracker run. + * + * The next time the metadata tracker runs the "continueState()" method applies the "hole retention" + * to state.getLastGoodTxCommitTimeInIndex(). This causes the state.getLastGoodTxCommitTimeInIndex() to scan + * for prior transactions that might have been missed. + * + */ + + Long fromCommitTime = getTxFromCommitTime(txnsFound, + state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime()); + log.debug("#### Check txnsFound : " + txnsFound.size()); + log.debug("======= fromCommitTime: " + fromCommitTime); + + // When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction + // for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime. + // When there isn't nodes in that range, -1 is returned as commit times + boolean shardOutOfRange = false; if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable) { - transactions = getDBIDRangeTransactions(fromCommitTime, txnsFound); - } - else - { - transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, - state.getTimeToStopIndexing()); + + DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter; + Pair commitTimes = client.getTxIntervalCommitTime(coreName, + dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); + Long shardMinCommitTime = commitTimes.getFirst(); + Long shardMaxCommitTime = commitTimes.getSecond(); + + // Node Range it's not still available in repository + if (shardMinCommitTime == -1) + { + log.debug( + "#### [DB_ID_RANGE] No nodes in range [{}-{}] " + + "exist in the repository. Indexing only latest transaction.", + dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); + shardOutOfRange = true; + } + if (fromCommitTime > shardMaxCommitTime) + { + log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. " + + "Indexing only latest transaction.", + dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange()); + shardOutOfRange = true; + } + // Initial commit time for Node Range is greater than calculated from commit time + if (fromCommitTime < shardMinCommitTime) + { + log.debug("#### [DB_ID_RANGE] Skipping transactions from {} to {}", + fromCommitTime, shardMinCommitTime); + fromCommitTime = shardMinCommitTime; + } } - // Remove transactions already indexed - transactions = removeIndexedTransactions(transactions); - - if (transactions.getTransactions().size() > 0) + log.debug("#### Get txn from commit time: " + fromCommitTime); + transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, + state.getTimeToStopIndexing()); + + + // When transactions are out of Shard range, only the latest transaction needs to be indexed + // in order to preserve the state up-to-date of the MetadataTracker + if (shardOutOfRange) { - LOGGER.info("{}-[CORE {}] Found {} transactions after lastTxCommitTime {}, transactions from {} to {}", - Thread.currentThread().getId(), - coreName, - transactions.getTransactions().size(), - fromCommitTime, - transactions.getTransactions().get(0), - transactions.getTransactions().get(transactions.getTransactions().size() - 1)); - } - else - { - LOGGER.info("{}-[CORE {}] No transaction found after lastTxCommitTime {}", - Thread.currentThread().getId(), - coreName, - ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime())); + Transaction latestTransaction = new Transaction(); + latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); + latestTransaction.setId(transactions.getMaxTxnId()); + transactions = new Transactions( + Arrays.asList(latestTransaction), + transactions.getMaxTxnCommitTime(), + transactions.getMaxTxnId()); + log.debug("#### [DB_ID_RANGE] Latest transaction to be indexed {}", latestTransaction); } - // Group the transactions in batches of transactionDocsBatchSize (or less) - List> txBatches = new ArrayList<>(); - List txBatch = new ArrayList<>(); + setLastTxCommitTimeAndTxIdInTrackerState(transactions, state); + + log.debug("Scanning transactions ..."); + if (transactions.getTransactions().size() > 0) { + log.info(".... from " + transactions.getTransactions().get(0)); + log.info(".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1)); + } else { + log.info(".... none found after lastTxCommitTime " + + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state + .getLastIndexedTxCommitTime())); + } + + ArrayList txBatch = new ArrayList<>(); for (Transaction info : transactions.getTransactions()) { - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}", - Thread.currentThread().getId(), coreName, transactions.getTransactions().size(), info.getId()); + /* + * isInIndex is used to ensure transactions that are being re-pulled due to "hole retention" are not re-indexed if + * they have already been indexed. + * + * The logic in infoSrv.txnInIndex() first checks an in-memory LRUcache for the txnId. If it doesn't find it in the cache + * it checks the index. The LRUCache is only needed for txnId's that have been indexed but are not yet visible in the index for + * one of two reasons: + * + * 1) The commit tracker has not yet committed the transaction. + * 2) The txnId has been committed to the index but the new searcher has not yet been warmed. + * + * This means that to ensure txnId's are not needlessly reprocessed during hole retention, the LRUCache must be large + * enough to cover the time between when a txnId is indexed and when it becomes visible. + */ + + boolean isInIndex = (infoSrv.txnInIndex(info.getId(), true) && info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime()); + if (isInIndex) { + txnsFound.add(info); + } else { + // Make sure we do not go ahead of where we started - we will check the holes here + // correctly next time + if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) { + upToDate = true; + break; + } + + txBatch.add(info); + if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) { + + docCount += indexBatchOfTransactions(txBatch); + totalUpdatedDocs += docCount; + + for (Transaction scheduledTx : txBatch) { + txnsFound.add(scheduledTx); + txsIndexed.add(scheduledTx); + } + txBatch.clear(); + } } - // Make sure we do not go ahead of where we started - we will check the holes here - // correctly next time - if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) { - upToDate = true; - break; - } + if (docCount > batchCount) { + indexTransactionsAfterAsynchronous(txsIndexed, state); + long endElapsed = System.nanoTime(); + trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); + startElapsed = endElapsed; + docCount = 0; - txBatch.add(info); - if (getUpdateAndDeleteCount(txBatch) > transactionDocsBatchSize) { - txBatches.add(txBatch); - txBatch = new ArrayList<>(); + //Release the write lock allowing the commit tracker to run. + this.getWriteLock().release(); + //Re-acquire the write lock and keep indexing. + this.getWriteLock().acquire(); } - } - if (!txBatch.isEmpty()) - { - txBatches.add(txBatch); + checkShutdown(); } - // Index batches of transactions and the nodes updated or deleted within the transaction - for (List batch : txBatches) - { + // Index any remaining transactions bringing the index to a consistent state so the CommitTracker can commit if need be. - // Index nodes contained in the transactions - int docCount = indexBatchOfTransactions(batch, totalUpdatedDocs); - totalUpdatedDocs += docCount; + if (!txBatch.isEmpty()) { + if (this.getUpdateAndDeleteCount(txBatch) > 0) { + docCount += indexBatchOfTransactions(txBatch); + totalUpdatedDocs += docCount; + } - // Add the transactions as found to avoid processing them again in the next iteration - batch.forEach(transaction -> txnsFound.add(transaction)); + for (Transaction scheduledTx : txBatch) { + txnsFound.add(scheduledTx); + txsIndexed.add(scheduledTx); + } + txBatch.clear(); + } - // Index the transactions - indexTransactionsAfterWorker(batch); + if (txsIndexed.size() > 0) { + indexTransactionsAfterAsynchronous(txsIndexed, state); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); startElapsed = endElapsed; + docCount = 0; } - - - - setLastTxCommitTimeAndTxIdInTrackerState(transactions); - } catch(Exception e) { @@ -1008,14 +918,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } while ((transactions.getTransactions().size() > 0) && (upToDate == false)); - LOGGER.info("{}-[CORE {}] Tracked {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs); + log.debug("total number of docs with metadata updated: " + totalUpdatedDocs); } - /** - * Update latest transaction indexed in MetadataTracker state - * @param transactions List of transactions indexed - */ - private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions) + private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions, TrackerState state) { Long maxTxnCommitTime = transactions.getMaxTxnCommitTime(); if (maxTxnCommitTime != null) @@ -1030,17 +936,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } } - /** - * Index transactions and update state of the tracker - * @param txsIndexed List of transactions to be indexed - * @throws IOException - */ - private void indexTransactionsAfterWorker(List txsIndexed) + private void indexTransactionsAfterAsynchronous(HashSet txsIndexed, TrackerState state) throws IOException { + waitForAsynchronous(); for (Transaction tx : txsIndexed) { - infoSrv.indexTransaction(tx, true); + super.infoSrv.indexTransaction(tx, true); // Transactions are ordered by commit time and tie-broken by tx id if (tx.getCommitTimeMs() > state.getLastIndexedTxCommitTime() || tx.getCommitTimeMs() == state.getLastIndexedTxCommitTime() @@ -1052,13 +954,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker trackerStats.addTxDocs((int) (tx.getDeletes() + tx.getUpdates())); } txsIndexed.clear(); + //super.infoSrv.commit(); } - /** - * Return the number of updated and deleted nodes in a list of transactions - * @param txs List of transactions - * @return Number of updated and deleted nodes - */ private long getUpdateAndDeleteCount(List txs) { long count = 0; @@ -1069,25 +967,12 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker return count; } - /** - * Index a batch of transactions. - * - * Updated or deleted nodes from these transactions are also packed into batches in order to get - * the metadata of the nodes in smaller invocations to Repository - * - * @param txBatch Batch of transactions to be indexed - * @param indexedNodes Number of nodes indexed in this Tracker execution - * - * @return Number of nodes indexed and last node indexed - * - * @throws AuthenticationException - * @throws IOException - * @throws JSONException - */ - private int indexBatchOfTransactions(List txBatch, int indexedNodes) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { - // Skip transactions without modifications (updates, deletes) - ArrayList nonEmptyTxs = new ArrayList<>(txBatch.size()); - ArrayList txIds = new ArrayList<>(); + private int indexBatchOfTransactions(List txBatch) throws AuthenticationException, IOException, JSONException + { + int nodeCount = 0; + List nonEmptyTxs = new ArrayList<>(txBatch.size()); + GetNodesParameters gnp = new GetNodesParameters(); + List txIds = new ArrayList<>(); for (Transaction tx : txBatch) { if (tx.getUpdates() > 0 || tx.getDeletes() > 0) @@ -1097,8 +982,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } } - // Get Nodes Id properties for every transaction - GetNodesParameters gnp = new GetNodesParameters(); gnp.setTransactionIds(txIds); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); @@ -1107,38 +990,42 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker gnp.setCoreName(coreName); List nodes = client.getNodes(gnp, Integer.MAX_VALUE); - - if (LOGGER.isDebugEnabled()) + + ArrayList nodeBatch = new ArrayList<>(); + for (Node node : nodes) { - LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}", Thread.currentThread().getId(), - coreName, nodes.size(), txIds); + if (log.isDebugEnabled()) + { + log.debug(node.toString()); + } + nodeBatch.add(node); + if (nodeBatch.size() > nodeBatchSize) + { + nodeCount += nodeBatch.size(); + NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv); + this.threadHandler.scheduleTask(niwr); + nodeBatch = new ArrayList<>(); + } } - - // Group the nodes in batches of nodeBatchSize (or less) - List> nodeBatches = Lists.partition(nodes, nodeBatchSize); - - Integer processedNodes = forkJoinPool.submit(() -> - nodeBatches.parallelStream().map(batch -> { - new NodeIndexWorker(batch, infoSrv).run(); - return batch.size(); - }).reduce(0, Integer::sum)).get(); - - - - return processedNodes; + + if (nodeBatch.size() > 0) + { + nodeCount += nodeBatch.size(); + NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv); + this.threadHandler.scheduleTask(niwr); + nodeBatch = new ArrayList<>(); + } + return nodeCount; } - - /** - * Node Indexing class running synchronously all the tracking operations. - */ - class NodeIndexWorker extends AbstractWorker + class NodeIndexWorkerRunnable extends AbstractWorkerRunnable { InformationServer infoServer; List nodes; - NodeIndexWorker(List nodes, InformationServer infoServer) + NodeIndexWorkerRunnable(QueueHandler queueHandler, List nodes, InformationServer infoServer) { + super(queueHandler); this.infoServer = infoServer; this.nodes = nodes; } @@ -1300,7 +1187,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention(); DO: do { - transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, endTime); + transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime); for (Transaction info : transactions.getTransactions()) { // include diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java index 196dc7b5d..946634c3c 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ModelTracker.java @@ -30,8 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Semaphore; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.alfresco.error.AlfrescoRuntimeException; @@ -49,8 +47,6 @@ import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.config.ConfigUtil; import org.apache.solr.core.SolrResourceLoader; import org.json.JSONException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @startuml @@ -86,7 +82,6 @@ import org.slf4j.LoggerFactory; */ public class ModelTracker extends AbstractTracker implements Tracker { - private static final Logger LOGGER = LoggerFactory.getLogger(ModelTracker.class); private final Set indexedStores = new HashSet<>(); private final Set ignoredStores = new HashSet<>(); @@ -105,36 +100,19 @@ public class ModelTracker extends AbstractTracker implements Tracker private volatile boolean hasModels = false; private File alfrescoModelDir; - // Share run and write locks across all ModelTracker threads - private static Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } - public ModelTracker(String solrHome, Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { super(p, client, coreName, informationServer, Tracker.Type.MODEL); String normalSolrHome = SolrResourceLoader.normalizeDir(solrHome); alfrescoModelDir = new File(ConfigUtil.locateProperty("solr.model.dir", normalSolrHome+"alfrescoModels")); - LOGGER.info("Alfresco Model dir {}", alfrescoModelDir); + logger.info("Alfresco Model dir " + alfrescoModelDir); if (!alfrescoModelDir.exists()) { alfrescoModelDir.mkdir(); } loadPersistedModels(); - - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } @Override @@ -219,7 +197,7 @@ public class ModelTracker extends AbstractTracker implements Tracker int registeredSearcherCount = this.infoSrv.getRegisteredSearcherCount(); if (registeredSearcherCount >= getMaxLiveSearchers()) { - LOGGER.info(".... skipping tracking registered searcher count = {}", registeredSearcherCount); + logger.info(".... skipping tracking registered searcher count = " + registeredSearcherCount); return; } @@ -290,7 +268,7 @@ public class ModelTracker extends AbstractTracker implements Tracker } catch (Throwable t) { - LOGGER.error("Model tracking failed for core: {}", coreName, t); + logger.error("Model tracking failed for core: "+ coreName, t); } } @@ -556,7 +534,7 @@ public class ModelTracker extends AbstractTracker implements Tracker { loadedModels.add(modelName); } - LOGGER.info("Loading model {}", model.getName()); + logger.info("Loading model " + model.getName()); } } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java similarity index 69% rename from search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java rename to search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java index d6a5d3c89..72834442b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoLockException.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/QueueHandler.java @@ -1,30 +1,24 @@ -/* - * Copyright (C) 2005-2020 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr; - -/** - * Marker exception thrown for indicating a failure in obtaining / releasing a lock. - */ -public class AlfrescoLockException extends Exception -{ - AlfrescoLockException(String message) - { - super(message); - } -} +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +public interface QueueHandler +{ + void removeFromQueueAndProdHead(AbstractWorkerRunnable job); +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java index 9ca1753f0..39e4ff3d2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SlaveCoreStatePublisher.java @@ -26,14 +26,9 @@ import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.TrackerState; import org.alfresco.solr.client.SOLRAPIClient; import org.apache.commons.codec.EncoderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Map; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Semaphore; /** * Despite belonging to the Tracker ecosystem, this component is actually a publisher, which periodically informs @@ -52,25 +47,6 @@ import java.util.concurrent.Semaphore; */ public class SlaveCoreStatePublisher extends CoreStatePublisher { - private static final Logger LOGGER = LoggerFactory.getLogger(SlaveCoreStatePublisher.class); - - - // Share run and write locks across all SlaveCoreStatePublisher threads - private static final Map RUN_LOCK_BY_CORE = new ConcurrentHashMap<>(); - private static final Map WRITE_LOCK_BY_CORE = new ConcurrentHashMap<>(); - - @Override - public Semaphore getWriteLock() - { - return WRITE_LOCK_BY_CORE.get(coreName); - } - - @Override - public Semaphore getRunLock() - { - return RUN_LOCK_BY_CORE.get(coreName); - } - public SlaveCoreStatePublisher( boolean isMaster, Properties coreProperties, @@ -79,9 +55,6 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher SolrInformationServer informationServer) { super(isMaster, coreProperties, repositoryClient, name, informationServer, NODE_STATE_PUBLISHER); - - RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); - WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } @Override @@ -94,7 +67,7 @@ public class SlaveCoreStatePublisher extends CoreStatePublisher } catch (EncoderException | IOException | AuthenticationException exception ) { - LOGGER.error("Unable to publish this node state. " + + logger.error("Unable to publish this node state. " + "A failure condition has been met during the outbound subscription message encoding process. " + "See the stacktrace below for further details.", exception); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java new file mode 100644 index 000000000..e22abf49f --- /dev/null +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ThreadHandler.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr.tracker; + +import java.util.Properties; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.alfresco.solr.tracker.pool.DefaultTrackerPoolFactory; +import org.alfresco.solr.tracker.pool.TrackerPoolFactory; + +/** + * This class handles threads for trackers. + * + * @author Ahmed Owian + */ +public class ThreadHandler implements QueueHandler +{ + /** the instance that will be given out by the factory */ + private ThreadPoolExecutor threadPool; + private LinkedBlockingQueue threadQueue = new LinkedBlockingQueue<>(); + private ReentrantReadWriteLock threadLock = new ReentrantReadWriteLock(true); + + public ThreadHandler(Properties p, String coreName, String trackerName) + { + // construct the instance + TrackerPoolFactory trackerPoolFactory = new DefaultTrackerPoolFactory(p, coreName, trackerName); + threadPool = trackerPoolFactory.create(); + } + + + public void scheduleTask(AbstractWorkerRunnable awr) + { + try + { + threadLock.writeLock().lock(); + // Add the runnable to the queue to ensure ordering + threadQueue.add(awr); + } + finally + { + threadLock.writeLock().unlock(); + } + threadPool.execute(awr); + } + + /** + * Removes the job from the queue and notifies the HEAD + */ + public void removeFromQueueAndProdHead(AbstractWorkerRunnable job) + { + try + { + threadLock.writeLock().lock(); + // Remove self from head of queue + threadQueue.remove(job); + } + finally + { + threadLock.writeLock().unlock(); + } + } + + /** + * Read-safe method to peek at the head of the queue + */ + public AbstractWorkerRunnable peekHeadReindexWorker() + { + try + { + threadLock.readLock().lock(); + return threadQueue.peek(); + } + finally + { + threadLock.readLock().unlock(); + } + } + + public void shutDownThreadPool() + { + if (threadPool != null) + { + threadPool.shutdownNow(); + } + } +} diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java index c5e90506c..d08d7d1ce 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/TrackerJob.java @@ -18,7 +18,6 @@ */ package org.alfresco.solr.tracker; -import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -29,12 +28,8 @@ import org.slf4j.LoggerFactory; * Generic Solr tracker job, allowing Quartz to initiate an index update from * a {@link Tracker} regardless of specific implementation. * - * Concurrent execution is disallowed, as no parallel work can be done when - * indexing contents from the repository. - * * @author Matt Ward */ -@DisallowConcurrentExecution public class TrackerJob implements Job { public static final String JOBDATA_TRACKER_KEY = "TRACKER"; diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java index ee5ac61e5..b8c91d370 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactory.java @@ -99,7 +99,7 @@ public class DefaultTrackerPoolFactory implements TrackerPoolFactory { case "AclTracker": corePoolSize = parseConfig("alfresco.acl.tracker.corePoolSize", p, corePoolSize); - maximumPoolSize = parseConfig("alfresco.acl.tracker.maximumPoolSize", p, maximumPoolSize); + maximumPoolSize = parseConfig("alfresco.acl.tracker.maximumPoolSize", p, maximumPoolSize); keepAliveTime = parseConfig("alfresco.acl.tracker.keepAliveTime", p, keepAliveTime); threadPriority = parseConfig("alfresco.acl.tracker.threadPriority", p, threadPriority); threadDaemon = parseConfigBoolean("alfresco.acl.tracker.threadDaemon", p); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java index 40586674f..0b3799a32 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/DocValueDocTransformer.java @@ -42,7 +42,11 @@ import org.slf4j.LoggerFactory; public class DocValueDocTransformer extends DocTransformer { protected final static Logger log = LoggerFactory.getLogger(DocValueDocTransformer.class); + + ResultContext context; + + /* (non-Javadoc) * @see org.apache.solr.response.transform.DocTransformer#getName() */ diff --git a/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java b/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java index 3951e7b8b..848381ca1 100644 --- a/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java +++ b/search-services/alfresco-search/src/main/java/org/apache/solr/core/CoreDescriptorDecorator.java @@ -43,7 +43,7 @@ public class CoreDescriptorDecorator private static String SECURE_COMMS_PROPERTY = "alfresco.secureComms"; - public static final ImmutableList SUBSTITUTABLE_PROPERTIES_SECURE = + public static ImmutableList substitutablePropertiesSecure = ImmutableList.of( "alfresco.host", "alfresco.port", @@ -59,7 +59,7 @@ public class CoreDescriptorDecorator "alfresco.encryption.ssl.keystore.provider", "alfresco.encryption.ssl.truststore.type"); - public static final ImmutableList SUBSTITUTABLE_PROPERTIES_NONE = + public static ImmutableList substitutablePropertiesNone = ImmutableList.of( "alfresco.host", "alfresco.port", @@ -74,11 +74,11 @@ public class CoreDescriptorDecorator String comms = ConfigUtil.locateProperty(SECURE_COMMS_PROPERTY, "none"); if (comms.equals("https")) { - coreProperties = SUBSTITUTABLE_PROPERTIES_SECURE; + coreProperties = substitutablePropertiesSecure; } else { - coreProperties = SUBSTITUTABLE_PROPERTIES_NONE; + coreProperties = substitutablePropertiesNone; } try diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 8cebb969b..4df0106b1 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -30,9 +30,6 @@ alfresco.hole.check.after=300000 alfresco.batch.count=5000 alfresco.recordUnindexedNodes=false -# max time (in msecs) a given tracker instance will try to acquire a lock on a given DBID -alfresco.tracker.maxNodeLockMs=120000 - # encryption # none, https @@ -122,22 +119,12 @@ solr.maxBooleanClauses=10000 # Batch fetch -#Max number of transactions fetched by metadata tracker -#alfresco.maxNumberOfTransactions= - -alfresco.transactionDocsBatchSize=2000 +alfresco.transactionDocsBatchSize=500 alfresco.nodeBatchSize=100 alfresco.changeSetAclsBatchSize=500 alfresco.aclBatchSize=100 alfresco.contentReadBatchSize=100 alfresco.contentUpdateBatchSize=1000 -alfresco.cascadeNodeBatchSize=10 - -# Trackers thread pools -#alfresco.metadataTrackerMaxParallelism= -#alfresco.aclTrackerMaxParallelism= -#alfresco.contentTrackerMaxParallelism= -#alfresco.cascadeTrackerMaxParallelism # Warming diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java index 390901404..bed47eeb4 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AbstractAlfrescoDistributedIT.java @@ -731,8 +731,7 @@ public abstract class AbstractAlfrescoDistributedIT extends SolrITInitializer public static void indexTransaction(Transaction transaction, List nodes, List nodeMetaDatas) { - // First map the nodes to a transaction. - nodes.stream().forEach(node -> node.setTxnId(transaction.getId())); + //First map the nodes to a transaction. SOLRAPIQueueClient.NODE_MAP.put(transaction.getId(), nodes); //Next map a node to the NodeMetaData diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java index 05bded1e4..aad132767 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrReloadIT.java @@ -68,7 +68,7 @@ public class AlfrescoSolrReloadIT extends AbstractAlfrescoSolrIT singletonList(acl), singletonList(aclReaders)); - int numNodes = 1000; + int numNodes = 1; List nodes = new ArrayList<>(); List nodeMetaDatas = new ArrayList<>(); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java new file mode 100644 index 000000000..ab1e2bdf7 --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/CloudTest.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2005-2014 Alfresco Software Limited. + * + * This file is part of Alfresco + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + */ +package org.alfresco.solr; + +public class CloudTest +{ + /* + public final static String QUERY = "a query"; + private Cloud cloud = new Cloud(); + @Mock SolrQueryRequest request; + + @SuppressWarnings({ "rawtypes" }) + @Test + public void testGetQueryWithZeroValues() + { + String fieldName = FIELD_DBID; + String operator = SolrInformationServer.AND; + Collection[] valueLists = new Collection[1]; + valueLists[0] = new ArrayList(); + String query = cloud.getQuery(fieldName, operator, valueLists); + assertEquals("", query); + + valueLists = new Collection[0]; + query = cloud.getQuery(fieldName, operator, valueLists); + assertEquals("", query); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testGetQueryWithOneValue() + { + String fieldName = FIELD_DBID; + String operator = SolrInformationServer.AND; + Collection[] valueLists = new Collection[1]; + valueLists[0] = new ArrayList(); + Object value = "value"; + valueLists[0].add(value); + String query = cloud.getQuery(fieldName, operator, valueLists); + assertEquals(FIELD_DBID + ":" + value, query); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Test + public void testGetQueryWithManyValues() + { + String fieldName = FIELD_DBID; + String operator = SolrInformationServer.AND; + Collection[] valueLists = new Collection[2]; + valueLists[0] = new ArrayList(); + Object value1 = "value1"; + valueLists[0].add(value1); + Object value2 = "value2"; + valueLists[0].add(value2); + Object value3 = "value3"; + valueLists[0].add(value3); + valueLists[1] = new ArrayList(); + Object value4 = "value4"; + valueLists[1].add(value4); + String query = cloud.getQuery(fieldName, operator, valueLists); + assertEquals(FIELD_DBID + ":" + value1 + operator + FIELD_DBID + ":" + value2 + operator + + FIELD_DBID + ":" + value3 + operator + FIELD_DBID + ":" + value4, query); + } + + @Test + public void testExists() + { + boolean exists = cloud.exists(super.selectRequestHandler, request, QUERY); + assertFalse(exists); + } + + @Test + public void testGetDocList() + { + // The response is created in the getDocList method, and the aftsRequestHandler is simply mocked. + // Therefore nothing is expected to be on the response for tests, so this verifies behavior. + DocList docList = cloud.getDocList(super.aftsRequestHandler, request, QUERY); + assertNull(docList); + + ArgumentCaptor response = ArgumentCaptor.forClass(SolrQueryResponse.class); + verify(super.aftsRequestHandler).handleRequest(eq(request), response.capture()); + assertNotNull(response.getValue()); + } + + @Test + public void testSelect() + { + SolrParams params = new ModifiableSolrParams(request.getParams()); + ResultContext rc = cloud.getResultContext(super.aftsRequestHandler, request, params); + assertNull(rc); + } + */ + +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index c4f8070dd..40bf8ac20 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -3,7 +3,6 @@ package org.alfresco.solr; import static org.alfresco.solr.AlfrescoSolrUtils.createCoreUsingTemplate; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import org.alfresco.solr.basics.RandomSupplier; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.apache.commons.io.FileUtils; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java index 6497d716f..1ca25cb1c 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/basics/SolrResponsesComparator.java @@ -1,6 +1,5 @@ package org.alfresco.solr.basics; -import com.google.common.collect.Sets; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -13,10 +12,8 @@ import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; public class SolrResponsesComparator { @@ -42,7 +39,7 @@ public class SolrResponsesComparator * Puts default values for handle */ public void putHandleDefaults() { - handle.put("[explain]", SKIPVAL); + handle.put("explain", SKIPVAL); handle.put("timestamp", SKIPVAL); handle.put("score", SKIPVAL); handle.put("wt", SKIP); @@ -54,7 +51,6 @@ public class SolrResponsesComparator handle.put("_version_", SKIP); handle.put("_original_parameters_", SKIP); handle.put("spellcheck-extras", SKIP); // No longer used can be removed in Solr 6. - handle.put("FIELDS", UNORDERED); } @@ -89,6 +85,27 @@ public class SolrResponsesComparator public void compareResponses(QueryResponse a, QueryResponse b) { + if (System.getProperty("remove.version.field") != null) + { + // we don't care if one has a version and the other doesnt - + // control vs distrib + // TODO: this should prob be done by adding an ignore on _version_ + // rather than mutating the responses? + if (a.getResults() != null) + { + for (SolrDocument doc : a.getResults()) + { + doc.removeFields("_version_"); + } + } + if (b.getResults() != null) + { + for (SolrDocument doc : b.getResults()) + { + doc.removeFields("_version_"); + } + } + } compareSolrResponses(a, b); } @@ -216,29 +233,6 @@ public class SolrResponsesComparator return compare1(b, a, flags, handle); } - public static String compare(Set a, Set b, int flags, Map handle) - { - String cmp; - cmp = compare1(a, b, flags, handle); - if (cmp != null) - return cmp; - return compare1(b, a, flags, handle); - } - - private static String compare1(Set a, Set b, int flags, Map handle) { - for (Object valA : a) - { - int flagsa = flags(handle, valA); - if ((flagsa & SKIP) != 0) - continue; - if (!b.contains(valA)) - { - return "[" + valA + "]==null"; - } - } - return null; - } - public static String compare(SolrDocument a, SolrDocument b, int flags, Map handle) { return compare(a.getFieldValuesMap(), b.getFieldValuesMap(), flags, handle); @@ -321,22 +315,10 @@ public class SolrResponsesComparator public static String compare(Object[] a, Object[] b, int flags, Map handle) { - - boolean ordered = (flags & UNORDERED) == 0; - if (a.length != b.length) { return ".length:" + a.length + "!=" + b.length; } - - if (!ordered) - { - Set setA = Sets.newHashSet(a); - Set setB = Sets.newHashSet(b); - return compare(setA, setB, flags, handle); - } - - for (int i = 0; i < a.length; i++) { String cmp = compare(a[i], b[i], flags, handle); @@ -346,7 +328,6 @@ public class SolrResponsesComparator return null; } - public static String compare(Object a, Object b, int flags, Map handle) { if (a == b) diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java index cdf81b87a..65266046b 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerExceptionIT.java @@ -18,8 +18,6 @@ */ package org.alfresco.solr.tracker; -import static java.util.Arrays.asList; - import static org.alfresco.solr.AlfrescoSolrUtils.ancestors; import static org.alfresco.solr.AlfrescoSolrUtils.createGUID; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; @@ -350,11 +348,13 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Fifteenth Test ##############################"); - // Change the aclReaders - aclReaders.setReaders(asList("andy")); + + List readers = aclReaders.getReaders(); + readers.set(0, "andy"); // Change the aclReader indexAclId(acl.getId()); - aclReaders2.setReaders(asList("ice")); + List readers2 = aclReaders2.getReaders(); + readers2.set(0, "ice"); // Change the aclReader reindexAclId(acl2.getId()); @@ -377,9 +377,8 @@ public class AlfrescoSolrTrackerExceptionIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Seventeenth Test ##############################"); - // Change the aclReaders - aclReaders.setReaders(asList("alan")); - aclReaders2.setReaders(asList("paul")); + readers.set(0, "alan"); // Change the aclReader + readers2.set(0, "paul"); // Change the aclReader reindexAclChangeSetId(aclChangeSet.getId()); //This should replace "andy" and "ice" with "alan" and "paul" diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java index df0d34c45..ca98f473d 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/AlfrescoSolrTrackerIT.java @@ -18,8 +18,6 @@ */ package org.alfresco.solr.tracker; -import static java.util.Arrays.asList; - import static org.alfresco.solr.AlfrescoSolrUtils.ancestors; import static org.alfresco.solr.AlfrescoSolrUtils.createGUID; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; @@ -355,11 +353,13 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Fifteenth Test ##############################"); - // Change the aclReaders - aclReaders.setReaders(asList("andy")); + + List readers = aclReaders.getReaders(); + readers.set(0, "andy"); // Change the aclReader indexAclId(acl.getId()); - aclReaders2.setReaders(asList("ice")); + List readers2 = aclReaders2.getReaders(); + readers2.set(0, "ice"); // Change the aclReader reindexAclId(acl2.getId()); @@ -382,9 +382,8 @@ public class AlfrescoSolrTrackerIT extends AbstractAlfrescoSolrIT logger.info("#################### Passed Seventeenth Test ##############################"); - // Change the aclReaders - aclReaders.setReaders(asList("alan")); - aclReaders2.setReaders(asList("paul")); + readers.set(0, "alan"); // Change the aclReader + readers2.set(0, "paul"); // Change the aclReader reindexAclChangeSetId(aclChangeSet.getId()); //This should replace "andy" and "ice" with "alan" and "paul" diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java index 650587546..c407684a1 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/ContentTrackerIT.java @@ -53,12 +53,14 @@ public class ContentTrackerIT private TrackerStats trackerStats; private int UPDATE_BATCH = 2; + private int READ_BATCH = 400; @Before public void setUp() throws Exception { doReturn("workspace://SpacesStore").when(props).getProperty(eq("alfresco.stores"), anyString()); doReturn("" + UPDATE_BATCH).when(props).getProperty(eq("alfresco.contentUpdateBatchSize"), anyString()); + doReturn("" + READ_BATCH).when(props).getProperty(eq("alfresco.contentReadBatchSize"), anyString()); when(srv.getTrackerStats()).thenReturn(trackerStats); this.contentTracker = new ContentTracker(props, repositoryClient, coreName, srv); @@ -101,14 +103,14 @@ public class ContentTrackerIT doc.tenant = "2"; docs2.add(doc); } - when(this.srv.getDocsWithUncleanContent()) + when(this.srv.getDocsWithUncleanContent(anyInt(), anyInt())) .thenReturn(docs1) .thenReturn(docs2) .thenReturn(emptyList); this.contentTracker.doTrack("anIterationId"); InOrder order = inOrder(srv); - order.verify(srv).getDocsWithUncleanContent(); + order.verify(srv).getDocsWithUncleanContent(0, READ_BATCH); /* * I had to make each bunch of calls have different parameters to prevent Mockito from incorrectly failing @@ -129,7 +131,7 @@ public class ContentTrackerIT order.verify(srv).updateContent(thirdDoc); order.verify(srv).commit(); - order.verify(srv).getDocsWithUncleanContent(); + order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH, READ_BATCH); // From docs2 docRef = new TenantDbId(); @@ -138,7 +140,7 @@ public class ContentTrackerIT order.verify(srv, times(UPDATE_BATCH)).updateContent(docRef); order.verify(srv).commit(); - order.verify(srv).getDocsWithUncleanContent(); + order.verify(srv).getDocsWithUncleanContent(0 + READ_BATCH + READ_BATCH, READ_BATCH); } @Test public void typeCheck() diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java index f3260eac9..55d18c26e 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedExpandDbidRangeAlfrescoSolrTrackerIT.java @@ -18,7 +18,6 @@ */ package org.alfresco.solr.tracker; -import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.solr.AbstractAlfrescoDistributedIT; import org.alfresco.solr.SolrInformationServer; import org.alfresco.solr.client.Acl; @@ -28,9 +27,6 @@ import org.alfresco.solr.client.Node; import org.alfresco.solr.client.NodeMetaData; import org.alfresco.solr.client.Transaction; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.TermQuery; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.util.NamedList; @@ -46,7 +42,6 @@ import java.util.Properties; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE; import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID; -import static org.alfresco.solr.AlfrescoSolrUtils.MAX_WAIT_TIME; import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; @@ -105,14 +100,6 @@ public class DistributedExpandDbidRangeAlfrescoSolrTrackerIT extends AbstractAlf bulkAcls, bulkAclReaders); - //Check for the ACL state stamp. - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST)); - builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, - bulkAclChangeSet.getId(), bulkAclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST)); - BooleanQuery waitForQuery = builder.build(); - waitForDocCount(waitForQuery, 1, MAX_WAIT_TIME); - SolrQueryResponse response0 = rangeCheck(0); NamedList values0 = response0.getValues(); //{start=0,end=100,nodeCount=0,maxDbid=0,density=NaN,expand=0,expanded=false} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java deleted file mode 100644 index e4236c7b9..000000000 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DocRouterFactoryTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2005-2014 Alfresco Software Limited. - * - * This file is part of Alfresco - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - */ -package org.alfresco.solr.tracker; - -import static org.alfresco.solr.tracker.DocRouterFactory.SHARD_RANGE_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.Properties; - -import org.alfresco.error.AlfrescoRuntimeException; -import org.alfresco.repo.index.shard.ShardMethodEnum; -import org.junit.Test; - -/** Unit tests for the {@link DocRouterFactory}. */ -public class DocRouterFactoryTest -{ - /** Check that a DB_ID_RANGE router can be created. */ - @Test - public void testDBIDRANGEWithShardRangeKey() - { - Properties mockProperties = mock(Properties.class); - when(mockProperties.containsKey(SHARD_RANGE_KEY)).thenReturn(true); - when(mockProperties.getProperty(SHARD_RANGE_KEY)).thenReturn("100000000-150000000"); - - // Call the method under test. - DocRouter docRouter = DocRouterFactory.getRouter(mockProperties, ShardMethodEnum.DB_ID_RANGE); - - assertTrue("Expected to get a DBIDRangeRouter.", docRouter instanceof DBIDRangeRouter); - DBIDRangeRouter dbidRangeRouter = (DBIDRangeRouter) docRouter; - assertEquals("Unexpected start of range.", dbidRangeRouter.getStartRange(), 100000000L); - assertEquals("Unexpected end of range.", dbidRangeRouter.getEndRange(), 150000000L); - } - - /** Check that an exception is raised if the range information is missing. */ - @Test(expected = AlfrescoRuntimeException.class) - public void testDBIDRANGEWithoutShardRangeKey() - { - Properties mockProperties = mock(Properties.class); - DocRouterFactory.getRouter(mockProperties, ShardMethodEnum.DB_ID_RANGE); - } -} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java index 641badb0b..98bb9d353 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/pool/DefaultTrackerPoolFactoryTest.java @@ -82,7 +82,7 @@ public class DefaultTrackerPoolFactoryTest public void testAclDefaultProperties() { poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker"); - + tpe = poolFactory.create(); assertEquals(4, tpe.getCorePoolSize()); assertEquals(4, tpe.getMaximumPoolSize()); @@ -96,7 +96,7 @@ public class DefaultTrackerPoolFactoryTest properties.put("alfresco.acl.tracker.keepAliveTime", "200"); poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "AclTracker"); tpe = poolFactory.create(); - + assertEquals(30, tpe.getCorePoolSize()); assertEquals(40, tpe.getMaximumPoolSize()); assertEquals(200, tpe.getKeepAliveTime(TimeUnit.SECONDS)); @@ -120,7 +120,7 @@ public class DefaultTrackerPoolFactoryTest properties.put("alfresco.content.tracker.keepAliveTime", "201"); poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "ContentTracker"); tpe = poolFactory.create(); - + assertEquals(100, tpe.getCorePoolSize()); assertEquals(140, tpe.getMaximumPoolSize()); assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS)); @@ -129,9 +129,9 @@ public class DefaultTrackerPoolFactoryTest public void testMetaDataDefaultProperties() { poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker"); - + tpe = poolFactory.create(); - + assertEquals(4, tpe.getCorePoolSize()); assertEquals(4, tpe.getMaximumPoolSize()); assertEquals(120, tpe.getKeepAliveTime(TimeUnit.SECONDS)); @@ -143,9 +143,9 @@ public class DefaultTrackerPoolFactoryTest properties.put("alfresco.metadata.tracker.maximumPoolSize", "140"); properties.put("alfresco.metadata.tracker.keepAliveTime", "201"); poolFactory = new DefaultTrackerPoolFactory(properties, "TheCore", "MetadataTracker"); - + tpe = poolFactory.create(); - + assertEquals(100, tpe.getCorePoolSize()); assertEquals(140, tpe.getMaximumPoolSize()); assertEquals(201, tpe.getKeepAliveTime(TimeUnit.SECONDS)); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java index b3969f120..4fd7e5bae 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerIT.java @@ -18,7 +18,6 @@ */ package org.alfresco.solr.transformer; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.alfresco.model.ContentModel; import org.alfresco.repo.search.adaptor.lucene.QueryConstants; import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter; @@ -82,7 +81,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute { putHandleDefaults(); //Test 1: Running a simple query without invoking AlfrescoFieldMapperTransformer, expected to see id,DBID and _version_ - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); assertEquals("Expecting 5 rows",5, results.size()); @@ -108,7 +107,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute putHandleDefaults(); //Test 2: Running simple query with AlfrescoFieldMapperTransformer, expected to see all fields returned - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","*,[fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","*,[fmap]")); SolrDocument docWithAllFields = resp.getResults().get(0); assertTrue(docWithAllFields.size() > 3); @@ -130,7 +129,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute putHandleDefaults(); //Test 3: Running simple query with AlfrescoFieldMapperTransformer, expected to see selected fields returned - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id,DBID,[fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id,DBID,[fmap]")); assertNotNull(resp); SolrDocument docWithRequestedFields = resp.getResults().get(0); @@ -145,7 +144,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute putHandleDefaults(); //Test 4: Running simple query with fmapDocTransformer on non default fields, expected to see selected fields returned - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap]")); assertNotNull(resp); SolrDocument docWithRequestedFields3 = resp.getResults().get(0); @@ -160,7 +159,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap], [fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","id, cm_title,[fmap], [fmap]")); assertNotNull(resp); SolrDocument docWithRequestedFields3 = resp.getResults().get(0); @@ -175,7 +174,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_name, score, [fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_name, score, [fmap]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); @@ -190,7 +189,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, cm_created, DBID, score, [fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, cm_created, DBID, score, [fmap]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); @@ -206,7 +205,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm?title, *name, [fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm?title, *name, [fmap]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); @@ -221,7 +220,7 @@ public class AlfrescoFieldMapperTransformerIT extends AbstractAlfrescoDistribute { putHandleDefaults(); - QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [explain], [fmap]", "sort", "id asc")); + QueryResponse resp = query(getDefaultTestClient(), true, ALFRESCO_JSON, params("q", "*", "qt", "/afts", "shards.qt", "/afts","fl","cm_title, [explain], [fmap]")); assertNotNull(resp); SolrDocumentList results = resp.getResults(); SolrDocument docWithAllFields = results.get(0); diff --git a/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java b/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java new file mode 100644 index 000000000..c936e639d --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/apache/lucene/analysis/minhash/MinHashFilterIT.java @@ -0,0 +1,567 @@ +package org.apache.lucene.analysis.minhash; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; +import org.apache.lucene.analysis.minhash.MinHashFilter.FixedSizeTreeSet; +import org.apache.lucene.analysis.minhash.MinHashFilter.LongPair; +import org.apache.lucene.analysis.shingle.ShingleFilterFactory; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.util.CharFilterFactory; +import org.apache.lucene.analysis.util.TokenFilterFactory; +import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; +import org.junit.Test; + +public class MinHashFilterIT extends BaseTokenStreamTestCase +{ + @Test + public void testIntHash() { + LongPair hash = new LongPair(); + MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(0), 0, 4, 0, hash); + assertEquals(-3485513579396041028L, hash.val1); + assertEquals(6383328099726337777L, hash.val2); + } + + @Test + public void testStringHash() throws UnsupportedEncodingException { + LongPair hash = new LongPair(); + byte[] bytes = "woof woof woof woof woof".getBytes("UTF-16LE"); + MinHashFilter.murmurhash3_x64_128(bytes, 0, bytes.length, 0, hash); + assertEquals(7638079586852243959L, hash.val1); + assertEquals(4378804943379391304L, hash.val2); + } + + @Test + public void testSimpleOrder() throws UnsupportedEncodingException { + LongPair hash1 = new LongPair(); + hash1.val1 = 1; + hash1.val2 = 2; + LongPair hash2 = new LongPair(); + hash2.val1 = 2; + hash2.val2 = 1; + assert (hash1.compareTo(hash2) > 0); + } + + + @Test + public void testHashOrder() { + assertTrue(!MinHashFilter.isLessThanUnsigned(0l, 0l)); + assertTrue(MinHashFilter.isLessThanUnsigned(0l, -1l)); + assertTrue(MinHashFilter.isLessThanUnsigned(1l, -1l)); + assertTrue(MinHashFilter.isLessThanUnsigned(-2l, -1l)); + assertTrue(MinHashFilter.isLessThanUnsigned(1l, 2l)); + assertTrue(MinHashFilter.isLessThanUnsigned(Long.MAX_VALUE, Long.MIN_VALUE)); + + FixedSizeTreeSet minSet = new FixedSizeTreeSet(500); + HashSet unadded = new HashSet(); + for (int i = 0; i < 100; i++) { + LongPair hash = new LongPair(); + MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash); + LongPair peek = null; + if (minSet.size() > 0) { + peek = minSet.last(); + } + + if (!minSet.add(hash)) { + unadded.add(hash); + } else { + if (peek != null) { + if ((minSet.size() == 500) && !peek.equals(minSet.last())) { + unadded.add(peek); + } + } + } + } + assertEquals(100, minSet.size()); + assertEquals(0, unadded.size()); + + HashSet collisionDetection = new HashSet(); + unadded = new HashSet(); + minSet = new FixedSizeTreeSet(500); + for (int i = 0; i < 1000000; i++) { + LongPair hash = new LongPair(); + MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash); + collisionDetection.add(hash); + LongPair peek = null; + if (minSet.size() > 0) { + peek = minSet.last(); + } + + if (!minSet.add(hash)) { + unadded.add(hash); + } else { + if (peek != null) { + if ((minSet.size() == 500) && !peek.equals(minSet.last())) { + unadded.add(peek); + } + } + } + } + assertEquals(1000000, collisionDetection.size()); + assertEquals(500, minSet.size()); + assertEquals(999500, unadded.size()); + + LongPair last = null; + LongPair current = null; + while ((current = minSet.pollLast()) != null) { + if (last != null) { + assertTrue(isLessThan(current, last)); + } else { + + } + last = current; + } + } + + + + @Test + public void testHashNotRepeated() { + FixedSizeTreeSet minSet = new FixedSizeTreeSet(500); + HashSet unadded = new HashSet(); + for (int i = 0; i < 10000; i++) { + LongPair hash = new LongPair(); + MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash); + LongPair peek = null; + if (minSet.size() > 0) { + peek = minSet.last(); + } + if (!minSet.add(hash)) { + unadded.add(hash); + } else { + if (peek != null) { + if ((minSet.size() == 500) && !peek.equals(minSet.last())) { + unadded.add(peek); + } + } + } + } + assertEquals(500, minSet.size()); + + LongPair last = null; + LongPair current = null; + while ((current = minSet.pollLast()) != null) { + if (last != null) { + assertTrue(isLessThan(current, last)); + } else { + + } + last = current; + } + } + + @Test + public void testMockShingleTokenizer() throws IOException { + Tokenizer mockShingleTokenizer = createMockShingleTokenizer(5, + "woof woof woof woof woof" + " " + "woof woof woof woof puff"); + assertTokenStreamContents(mockShingleTokenizer, + new String[] {"woof woof woof woof woof", "woof woof woof woof puff"}); + } + + @Test + public void testTokenStreamSingleInput() throws IOException { + String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯"}; + TokenStream ts = createTokenStream(5, "woof woof woof woof woof", 1, 1, 100, false); + assertTokenStreamContents(ts, hashes, new int[] {0}, + new int[] {24}, new String[] {MinHashFilter.MIN_HASH_TYPE}, new int[] {1}, new int[] {1}, 24, 0, null, + true); + + ts = createTokenStream(5, "woof woof woof woof woof", 2, 1, 1, false); + assertTokenStreamContents(ts, new String[] {new String(new char[] {0, 0, 8449, 54077, 64133, 32857, 8605, 41409}), + new String(new char[] {0, 1, 16887, 58164, 39536, 14926, 6529, 17276})}, new int[] {0, 0}, + new int[] {24, 24}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0}, new int[] {1, 1}, 24, 0, null, + true); + } + + @Test + public void testTokenStream1() throws IOException { + String[] hashes = new String[] {"℁팽徭聙↝ꇁ홱杯", + new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})}; + + TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 1, 1, 100,false); + assertTokenStreamContents(ts, hashes, new int[] {0, 0}, + new int[] {49, 49}, new String[] {MinHashFilter.MIN_HASH_TYPE, MinHashFilter.MIN_HASH_TYPE}, new int[] {1, 0}, + new int[] {1, 1}, 49, 0, null, true); + } + + private ArrayList getTokens(TokenStream ts) throws IOException { + ArrayList tokens = new ArrayList(); + ts.reset(); + while (ts.incrementToken()) { + CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class); + String token = new String(termAttribute.buffer(), 0, termAttribute.length()); + tokens.add(token); + } + ts.end(); + ts.close(); + + return tokens; + } + + private ArrayList getTokens(Analyzer analyzer, String field, String value) throws IOException + { + ArrayList tokens = new ArrayList(); + + TokenStream ts = analyzer.tokenStream(field, value); + ts.reset(); + while(ts.incrementToken()) + { + CharTermAttribute termAttribute = ts.getAttribute(CharTermAttribute.class); + String token = new String(termAttribute.buffer(), 0, termAttribute.length()); + tokens.add(token); + } + ts.end(); + ts.close(); + + return tokens; + } + + @Test + public void testTokenStream2() throws IOException { + TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 100, 1, 1, false); + ArrayList tokens = getTokens(ts); + ts.close(); + + assertEquals(100, tokens.size()); + } + + @Test + public void testTokenStream3() throws IOException { + TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 1, 10, false); + ArrayList tokens = getTokens(ts); + ts.close(); + + assertEquals(20, tokens.size()); + } + + @Test + public void testTokenStream4() throws IOException { + TokenStream ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, false); + ArrayList tokens = getTokens(ts); + ts.close(); + + assertEquals(20, tokens.size()); + + ts = createTokenStream(5, "woof woof woof woof woof" + " " + "woof woof woof woof puff", 10, 10, 1, true); + tokens = getTokens(ts); + ts.close(); + + assertEquals(100, tokens.size()); + + } + + @Test + public void testLSHQuery() throws IOException + { + Analyzer analyzer = createMinHashAnalyzer(5, 1, 100); + IndexWriterConfig config = new IndexWriterConfig(analyzer); + + RAMDirectory directory = new RAMDirectory(); + IndexWriter writer = new IndexWriter(directory, config); + Document doc = new Document(); + doc.add(new TextField("text", "woof woof woof woof woof", Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("text", "woof woof woof woof woof puff", Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("text", "woof woof woof woof puff", Store.NO)); + writer.addDocument(doc); + + writer.commit(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", "℁팽徭聙↝ꇁ홱杯"))), Occur.SHOULD); + builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", new String(new char[] {36347, 63457, 43013, 56843, 52284, 34231, 57934, 42302})))), Occur.SHOULD); + builder.setDisableCoord(true); + TopDocs topDocs = searcher.search(builder.build(), 10); + + assertEquals(3, topDocs.totalHits); + + float score = topDocs.scoreDocs[0].score; + assertEquals(topDocs.scoreDocs[1].score, score/2, 0f); + assertEquals(topDocs.scoreDocs[2].score, score/2, 0f); + + } + + + + @Test + public void testLSHQuery2() throws IOException + { + String[] parts = new String[]{"one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"}; + int min = 5; + + Analyzer analyzer = createMinHashAnalyzer(min, 1, 100); + IndexWriterConfig config = new IndexWriterConfig(analyzer); + + RAMDirectory directory = new RAMDirectory(); + IndexWriter writer = new IndexWriter(directory, config); + + for(int i = 0; i < parts.length; i++) + { + StringBuilder builder = new StringBuilder(); + for(int j = 0; j < parts.length -i; j++) + { + if(builder.length() > 0) + { + builder.append(" "); + } + builder.append(parts[i+j]); + if(j >= min -1) + { + Document doc = new Document(); + doc.add(new TextField("text", builder.toString(), Store.NO)); + writer.addDocument(doc); + } + } + } + + writer.commit(); + writer.close(); + + + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); + + TopDocs topDocs = searcher.search(buildQuery("text", "one two three four five", min, 1, 100), 100); + assertEquals(6, topDocs.totalHits); + assertAllScores(topDocs, 1.0f); + topDocs = searcher.search(buildQuery("text", "two three four five six", min, 1, 100), 100); + assertEquals(10, topDocs.totalHits); + assertAllScores(topDocs, 1.0f); + topDocs = searcher.search(buildQuery("text", "three four five six seven", min, 1, 100), 100); + assertEquals(12, topDocs.totalHits); + assertAllScores(topDocs, 1.0f); + topDocs = searcher.search(buildQuery("text", "four five six seven eight", min, 1, 100), 100); + assertEquals(12, topDocs.totalHits); + assertAllScores(topDocs, 1.0f); + topDocs = searcher.search(buildQuery("text", "five six seven eight nine", min, 1, 100), 100); + assertEquals(10, topDocs.totalHits); + assertAllScores(topDocs, 1.0f); + topDocs = searcher.search(buildQuery("text", "six seven eight nine ten", min, 1, 100), 100); + assertEquals(6, topDocs.totalHits); + assertAllScores(topDocs, 1.0f); + + topDocs = searcher.search(buildQuery("text", "one two three four five six", min, 1, 100), 100); + assertEquals(11, topDocs.totalHits); + + topDocs = searcher.search(buildQuery("text", "one two three four five six seven eight nine ten", min, 1, 100), 100); + assertEquals(21, topDocs.totalHits); + for(int i = 0; i < topDocs.totalHits; i++) + { + System.out.println(i+" = "+topDocs.scoreDocs[i]); + } + + float topScore = 6.0f; + assertEquals(topDocs.scoreDocs[0].score, topScore, 0.001f); + assertEquals(topDocs.scoreDocs[1].score, topScore * 5/6, 0.001f); + assertEquals(topDocs.scoreDocs[2].score, topScore * 5/6, 0.001f); + assertEquals(topDocs.scoreDocs[3].score, topScore * 4/6, 0.001f); + assertEquals(topDocs.scoreDocs[4].score, topScore * 4/6, 0.001f); + assertEquals(topDocs.scoreDocs[5].score, topScore * 4/6, 0.001f); + assertEquals(topDocs.scoreDocs[6].score, topScore * 3/6, 0.001f); + assertEquals(topDocs.scoreDocs[7].score, topScore * 3/6, 0.001f); + assertEquals(topDocs.scoreDocs[8].score, topScore * 3/6, 0.001f); + assertEquals(topDocs.scoreDocs[9].score, topScore * 3/6, 0.001f); + assertEquals(topDocs.scoreDocs[10].score, topScore * 2/6, 0.001f); + assertEquals(topDocs.scoreDocs[11].score, topScore * 2/6, 0.001f); + assertEquals(topDocs.scoreDocs[12].score, topScore * 2/6, 0.001f); + assertEquals(topDocs.scoreDocs[13].score, topScore * 2/6, 0.001f); + assertEquals(topDocs.scoreDocs[14].score, topScore * 2/6, 0.001f); + assertEquals(topDocs.scoreDocs[15].score, topScore * 1/6, 0.001f); + assertEquals(topDocs.scoreDocs[16].score, topScore * 1/6, 0.001f); + assertEquals(topDocs.scoreDocs[17].score, topScore * 1/6, 0.001f); + assertEquals(topDocs.scoreDocs[18].score, topScore * 1/6, 0.001f); + assertEquals(topDocs.scoreDocs[19].score, topScore * 1/6, 0.001f); + assertEquals(topDocs.scoreDocs[20].score, topScore * 1/6, 0.001f); + + } + + + private void assertAllScores(TopDocs topDocs, float score) + { + for(int i = 0; i < topDocs.totalHits; i++) + { + assertEquals(topDocs.scoreDocs[i].score, score, 0f); + } + } + + private Query buildQuery(String field, String query, int min, int hashCount, int hashSetSize) throws IOException + { + TokenizerChain chain = createMinHashAnalyzer(min, hashCount, hashSetSize); + ArrayList tokens = getTokens(chain, field, query); + chain.close(); + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for(String token : tokens) + { + builder.add(new ConstantScoreQuery(new TermQuery(new Term("text", token))), Occur.SHOULD); + } + builder.setDisableCoord(true); + return builder.build(); + } + + public static TokenStream createTokenStream(int shingleSize, String shingles, int hashCount, int bucketCount, int hashSetSize, boolean withRotation) { + Tokenizer tokenizer = createMockShingleTokenizer(shingleSize, shingles); + HashMap lshffargs = new HashMap(); + lshffargs.put("hashCount", "" + hashCount); + lshffargs.put("bucketCount", "" + bucketCount); + lshffargs.put("hashSetSize", "" + hashSetSize); + lshffargs.put("withRotation", "" + withRotation); + MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs); + return lshff.create(tokenizer); + } + + public static TokenizerChain createMinHashAnalyzer(int min, int hashCount, int hashSetSize) + { + WhitespaceTokenizerFactory icutf = new WhitespaceTokenizerFactory(Collections.emptyMap()); + HashMap sffargs = new HashMap(); + sffargs.put("minShingleSize", ""+min); + sffargs.put("maxShingleSize", ""+min); + sffargs.put("outputUnigrams", "false"); + sffargs.put("outputUnigramsIfNoShingles", "false"); + sffargs.put("tokenSeparator", " "); + ShingleFilterFactory sff = new ShingleFilterFactory(sffargs); + HashMap lshffargs = new HashMap(); + lshffargs.put("hashCount", ""+hashCount); + lshffargs.put("hashSetSize", ""+hashSetSize); + MinHashFilterFactory lshff = new MinHashFilterFactory(lshffargs); + + TokenizerChain chain = new TokenizerChain(new CharFilterFactory[]{}, icutf, new TokenFilterFactory[]{sff, lshff}); + return chain; + } + + public static Tokenizer createMockShingleTokenizer(int shingleSize, String shingles) { + MockTokenizer tokenizer = new MockTokenizer( + new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+([ \t\r\n]+[^ \t\r\n]+){4}").toAutomaton()), + true); + tokenizer.setEnableChecks(true); + if (shingles != null) { + tokenizer.setReader(new StringReader(shingles)); + } + return tokenizer; + } + + private boolean isLessThan(LongPair hash1, LongPair hash2) { + if (MinHashFilter.isLessThanUnsigned(hash1.val2, hash2.val2)) { + return true; + } else if (hash1.val2 == hash2.val2) { + return (MinHashFilter.isLessThanUnsigned(hash1.val1, hash2.val1)); + } else { + return false; + } + } + + + /** + * An analyzer that uses a tokenizer and a list of token filters to + * create a TokenStream - lifted from SOLR to make this analyzer test lucene only. + */ + public static class TokenizerChain extends Analyzer { + + final private CharFilterFactory[] charFilters; + final private TokenizerFactory tokenizer; + final private TokenFilterFactory[] filters; + + + /** + * Creates a new TokenizerChain. + * + * @param charFilters Factories for the CharFilters to use, if any - if null, will be treated as if empty. + * @param tokenizer Factory for the Tokenizer to use, must not be null. + * @param filters Factories for the TokenFilters to use if any- if null, will be treated as if empty. + */ + public TokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) { + this.charFilters = charFilters; + this.tokenizer = tokenizer; + this.filters = filters; + } + + @Override + public Reader initReader(String fieldName, Reader reader) { + if (charFilters != null && charFilters.length > 0) { + Reader cs = reader; + for (CharFilterFactory charFilter : charFilters) { + cs = charFilter.create(cs); + } + reader = cs; + } + return reader; + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tk = tokenizer.create(); + TokenStream ts = tk; + for (TokenFilterFactory filter : filters) { + ts = filter.create(ts); + } + return new TokenStreamComponents(tk, ts); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TokenizerChain("); + for (CharFilterFactory filter: charFilters) { + sb.append(filter); + sb.append(", "); + } + sb.append(tokenizer); + for (TokenFilterFactory filter: filters) { + sb.append(", "); + sb.append(filter); + } + sb.append(')'); + return sb.toString(); + } + } +} diff --git a/search-services/alfresco-search/src/test/resources/log4j.xml b/search-services/alfresco-search/src/test/resources/log4j.xml index ff8133825..be048ddb9 100644 --- a/search-services/alfresco-search/src/test/resources/log4j.xml +++ b/search-services/alfresco-search/src/test/resources/log4j.xml @@ -7,6 +7,11 @@ + + + + + + + + + +